You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2017/01/24 22:05:43 UTC
ambari git commit: AMBARI-19660. Add log rotation settings - handle
upgrade scenario - Storm, Hive (Madhuvanthi Radhakrishnan via smohanty)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 c34a836b7 -> 9f8401a4a
AMBARI-19660. Add log rotation settings - handle upgrade scenario - Storm, Hive (Madhuvanthi Radhakrishnan via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9f8401a4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9f8401a4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9f8401a4
Branch: refs/heads/branch-2.5
Commit: 9f8401a4af4bdb123c030d7f783a5176136ec5db
Parents: c34a836
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Jan 24 14:05:34 2017 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Jan 24 14:05:34 2017 -0800
----------------------------------------------------------------------
.../stacks/HDP/2.3/upgrades/config-upgrade.xml | 46 +++++++++++++++++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 27 +++++++++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 29 ++++++++++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 28 ++++++++++++
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 6 ++-
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 5 +++
.../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 45 ++++++++++++++++++-
.../stacks/HDP/2.4/upgrades/config-upgrade.xml | 44 ++++++++++++++++++
.../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 28 ++++++++++++
.../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 27 +++++++++++
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 9 +++-
.../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 8 ++++
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 47 ++++++++++++++++++++
.../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 30 ++++++++++++-
.../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 8 ++++
15 files changed, 382 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index 09de45c..1974acb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -177,6 +177,21 @@
<transfer operation="delete" delete-key="atlas.hook.hive.maxThreads" />
</definition>
+ <definition xsi:type="configure" id="hive_log4j_parameterize" summary="Parameterizing Hive Log4J Properties">
+ <type>hive-log4j</type>
+ <set key="hive_log_maxfilesize" value="256"/>
+ <set key = "hive_log_maxbackupindex" value="30"/>
+ <regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/>
+ <replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/>
+ </definition>
+ <definition xsi:type="configure" id="hive_llap_log4j_parameterize" summary="Parameterizing Hive llap Log4J Properties">
+ <type>llap-daemon-log4j</type>
+ <set key="hive_llap_log_maxfilesize" value="256"/>
+ <set key = "hive_llap_log_maxbackupindex" value="240"/>
+ <regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/>
+ <regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/>
+ </definition>
+
</changes>
</component>
<component name="WEBHCAT_SERVER">
@@ -198,6 +213,14 @@
<replace key="templeton.hcat" find="/usr/hdp/current/hive-client" replace-with="/usr/hdp/${hdp.version}/hive"/>
<set key="templeton.hive.extra.files" value="/usr/hdp/${hdp.version}/tez/conf/tez-site.xml,/usr/hdp/${hdp.version}/tez,/usr/hdp/${hdp.version}/tez/lib"/>
</definition>
+
+ <definition xsi:type="configure" id="webhcat_log4j_parameterize" summary="Parameterizing Webhcat Log4J Properties">
+ <type>webhcat-log4j</type>
+ <set key="webhcat_log_maxfilesize" value="256"/>
+ <set key = "webhcat_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxFileSize = {{webhcat_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxBackupIndex = {{webhcat_log_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -549,6 +572,29 @@
if-type="storm-site"
if-value="15000" />
</definition>
+
+ <definition xsi:type="configure" id="storm_worker_log4j_parameterize" summary="Parameterizing Storm Worker Log4J Properties">
+ <type>storm-worker-log4j</type>
+ <set key="storm_wrkr_a1_maxfilesize" value="100"/>
+ <set key="storm_wrkr_a1_maxbackupindex" value="9"/>
+ <set key="storm_wrkr_out_maxfilesize" value="100"/>
+ <set key="storm_wrkr_out_maxbackupindex" value="4"/>
+ <set key="storm_wrkr_err_maxfilesize" value="100"/>
+ <set key="storm_wrkr_err_maxbackupindex" value="4"/>
+ <regex-replace key="content" find="A1"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="A1"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/>
+ <regex-replace key="content" find="STDOUT"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="STDOUT"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/>
+ <regex-replace key="content" find="STDERR"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="STDERR"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/>
+ </definition>
+ <definition xsi:type="configure" id="storm_cluster_log4j_parameterize" summary="Parameterizing Storm Cluster Log4J Properties">
+ <type>storm-cluster-log4j</type>
+ <set key="storm_a1_maxfilesize" value="100"/>
+ <set key="storm_a1_maxbackupindex" value="9"/>
+ <regex-replace key="content" find="A1" immediateFlush="false"
 fileName="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
 filePattern="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="A1" immediateFlush="false"
 fileName="${sys:storm.log.dir}/${sys:logfile.name}"
 filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
index 81c470a..619fa5d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
@@ -318,6 +318,22 @@
<task xsi:type="configure" id="hdp_2_4_0_0_webhcat_server_update_configuration_paths"/>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties">
+ <task xsi:type="configure" id="hive_log4j_parameterize">
+ <summary>Updating the Hive Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties">
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize">
+ <summary>Updating the Hive llap Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties">
+ <task xsi:type="configure" id="webhcat_log4j_parameterize">
+ <summary>Updating the Webhcat Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/>
</execute-stage>
@@ -368,6 +384,17 @@
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
</execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties">
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize">
+ <summary>Updating the Storm worker Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties">
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize">
+ <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!--ZOOKEEPER-->
<execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Parameterizing Zookeeper Log4J Properties">
<task xsi:type="configure" id="zookeeper_log4j_parameterize">
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index 3e61aad..2aedfa8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -362,6 +362,24 @@
<task xsi:type="configure" id="hdp_2_5_0_0_webhcat_server_update_configuration_paths"/>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties">
+ <task xsi:type="configure" id="hive_log4j_parameterize">
+ <summary>Updating the Hive Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties">
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize">
+ <summary>Updating the Hive llap Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties">
+ <task xsi:type="configure" id="webhcat_log4j_parameterize">
+ <summary>Updating the Webhcat Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/>
</execute-stage>
@@ -557,6 +575,17 @@
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
</execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties">
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize">
+ <summary>Updating the Storm worker Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties">
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize">
+ <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!--ATLAS-->
<execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties">
<task xsi:type="configure" id="atlas_log4j_parameterize">
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
index cffecaa3..dafe8e3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
@@ -364,6 +364,12 @@
<task xsi:type="configure" id="hdp_2_5_0_0_webhcat_server_update_configuration_paths"/>
</execute-stage>
+ <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties">
+ <task xsi:type="configure" id="webhcat_log4j_parameterize">
+ <summary>Updating the Webhcat Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/>
</execute-stage>
@@ -383,6 +389,17 @@
</task>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties">
+ <task xsi:type="configure" id="hive_log4j_parameterize">
+ <summary>Updating the Hive Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties">
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize">
+ <summary>Updating the Hive llap Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!--OOZIE-->
<execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation">
@@ -560,6 +577,17 @@
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
</execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties">
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize">
+ <summary>Updating the Storm worker Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties">
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize">
+ <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!--ATLAS-->
<execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties">
<task xsi:type="configure" id="atlas_log4j_parameterize">
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 5ae6876..41a3eda 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -752,6 +752,8 @@
<component name="HIVE_SERVER">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/>
+ <task xsi:type="configure" id="hive_log4j_parameterize" />
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade/>
@@ -764,6 +766,7 @@
<component name="WEBHCAT_SERVER">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_4_0_0_webhcat_server_update_configuration_paths"/>
+ <task xsi:type="configure" id="webhcat_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade/>
@@ -931,7 +934,8 @@
<component name="NIMBUS">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_3_0_0_remove_empty_storm_topology_submission_notifier_plugin_class"/>
-
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize" />
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize" />
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
</pre-upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index eafb129..df54835 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -874,6 +874,8 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_4_0_0_hive_server_configure_authentication"/>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hive_audit_db" />
+ <task xsi:type="configure" id="hive_log4j_parameterize" />
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize" />
<!-- Remove Atlas configs that were incorrectly added to hive-site instead of Atlas' application.properties. -->
<task xsi:type="configure" id="hdp_2_5_0_0_remove_hive_atlas_configs" />
@@ -893,6 +895,7 @@
<component name="WEBHCAT_SERVER">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_webhcat_server_update_configuration_paths"/>
+ <task xsi:type="configure" id="webhcat_log4j_parameterize"/>
</pre-upgrade>
<pre-downgrade />
@@ -1094,6 +1097,8 @@
<function>delete_storm_local_data</function>
</task>
<task xsi:type="configure" id="hdp_2_5_0_0_add_storm_security_configs" />
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize" />
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
index 8101682..981947f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
@@ -559,6 +559,10 @@
<processing>
<service name="ZOOKEEPER">
<component name="ZOOKEEPER_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="zookeeper_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade />
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -612,6 +616,7 @@
<script>scripts/ranger_admin.py</script>
<function>setup_ranger_java_patches</function>
</task>
+ <task xsi:type="configure" id="admin_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade/> <!-- no-op to prevent config changes on downgrade -->
@@ -623,6 +628,19 @@
</component>
<component name="RANGER_USERSYNC">
+ <pre-upgrade>
+ <task xsi:type="configure" id="usersync_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade />
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ <component name="RANGER_TAGSYNC">
+ <pre-upgrade>
+ <task xsi:type="configure" id="tagsync_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade />
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -633,7 +651,7 @@
<component name="RANGER_KMS_SERVER">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db" />
-
+ <task xsi:type="configure" id="kms_log4j_parameterize" />
<task xsi:type="execute" hosts="any" sequential="true">
<summary>Upgrading Ranger KMS database schema</summary>
<script>scripts/kms_server.py</script>
@@ -660,6 +678,7 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" />
+ <task xsi:type="configure" id="hdfs_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
@@ -739,6 +758,7 @@
<summary>Calculating Yarn Properties for Spark Shuffle</summary>
</task>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db" />
+ <task xsi:type="configure" id="yarn_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
@@ -772,6 +792,7 @@
<!-- These HBASE configs changed in HDP 2.3.4.0, but Ambari can't distinguish HDP 2.3.2.0 vs HDP 2.3.4.0, so easier to always do them. -->
<task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db" />
+ <task xsi:type="configure" id="hbase_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
@@ -861,6 +882,8 @@
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HiveEnvClasspathAction">
<summary>Update hive-env content</summary>
</task>
+ <task xsi:type="configure" id="hive_log4j_parameterize" />
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade/>
@@ -873,6 +896,7 @@
<component name="WEBHCAT_SERVER">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_webhcat_server_update_configuration_paths"/>
+ <task xsi:type="configure" id="webhcat_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade />
@@ -946,7 +970,7 @@
<component name="OOZIE_SERVER">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
-
+ <task xsi:type="configure" id="oozie_log4j_parameterize" />
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation">
<summary>Adjusting Oozie properties</summary>
</task>
@@ -1000,6 +1024,7 @@
<component name="FALCON_SERVER">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"/>
+ <task xsi:type="configure" id="falcon_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade/>
<upgrade>
@@ -1018,6 +1043,7 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db" />
<task xsi:type="configure" id="hdp_2_5_0_0_add_protocol_compatibility" />
+ <task xsi:type="configure" id="kafka_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade/>
@@ -1032,6 +1058,8 @@
<component name="KNOX_GATEWAY">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_knox_audit_db" />
+ <task xsi:type="configure" id="knox_ldap_log4j_parameterize" />
+ <task xsi:type="configure" id="knox_gateway_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
@@ -1067,6 +1095,8 @@
<function>delete_storm_local_data</function>
</task>
<task xsi:type="configure" id="hdp_2_5_0_0_add_storm_security_configs" />
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize" />
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade>
@@ -1192,5 +1222,16 @@
</upgrade>
</component>
</service>
+ <service name="ATLAS">
+ <component name="ATLAS_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="atlas_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade />
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
</processing>
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index db848c1..e6817f6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -78,6 +78,21 @@
<transfer operation="delete" delete-key="atlas.hook.hive.maxThreads" />
</definition>
+ <definition xsi:type="configure" id="hive_log4j_parameterize" summary="Parameterizing Hive Log4J Properties">
+ <type>hive-log4j</type>
+ <set key="hive_log_maxfilesize" value="256"/>
+ <set key = "hive_log_maxbackupindex" value="30"/>
+ <regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/>
+ <replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/>
+ </definition>
+ <definition xsi:type="configure" id="hive_llap_log4j_parameterize" summary="Parameterizing Hive llap Log4J Properties">
+ <type>llap-daemon-log4j</type>
+ <set key="hive_llap_log_maxfilesize" value="256"/>
+ <set key = "hive_llap_log_maxbackupindex" value="240"/>
+ <regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/>
+ <regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/>
+ </definition>
+
</changes>
</component>
@@ -91,6 +106,13 @@
<replace key="templeton.hcat" find="/usr/hdp/current/hive-client" replace-with="/usr/hdp/${hdp.version}/hive"/>
<set key="templeton.hive.extra.files" value="/usr/hdp/${hdp.version}/tez/conf/tez-site.xml,/usr/hdp/${hdp.version}/tez,/usr/hdp/${hdp.version}/tez/lib"/>
</definition>
+ <definition xsi:type="configure" id="webhcat_log4j_parameterize" summary="Parameterizing Webhcat Log4J Properties">
+ <type>webhcat-log4j</type>
+ <set key="webhcat_log_maxfilesize" value="256"/>
+ <set key = "webhcat_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxFileSize = {{webhcat_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxBackupIndex = {{webhcat_log_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -381,6 +403,28 @@
if-type="storm-site"
if-value="15000" />
</definition>
+ <definition xsi:type="configure" id="storm_worker_log4j_parameterize" summary="Parameterizing Storm Worker Log4J Properties">
+ <type>storm-worker-log4j</type>
+ <set key="storm_wrkr_a1_maxfilesize" value="100"/>
+ <set key="storm_wrkr_a1_maxbackupindex" value="9"/>
+ <set key="storm_wrkr_out_maxfilesize" value="100"/>
+ <set key="storm_wrkr_out_maxbackupindex" value="4"/>
+ <set key="storm_wrkr_err_maxfilesize" value="100"/>
+ <set key="storm_wrkr_err_maxbackupindex" value="4"/>
+ <regex-replace key="content" find="A1"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="A1"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/>
+ <regex-replace key="content" find="STDOUT"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="STDOUT"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/>
+ <regex-replace key="content" find="STDERR"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="STDERR"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/>
+ </definition>
+ <definition xsi:type="configure" id="storm_cluster_log4j_parameterize" summary="Parameterizing Storm Cluster Log4J Properties">
+ <type>storm-cluster-log4j</type>
+ <set key="storm_a1_maxfilesize" value="100"/>
+ <set key="storm_a1_maxbackupindex" value="9"/>
+ <regex-replace key="content" find="A1" immediateFlush="false"
 fileName="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
 filePattern="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="A1" immediateFlush="false"
 fileName="${sys:storm.log.dir}/${sys:logfile.name}"
 filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
index 1c4325f..34e56d3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
@@ -423,6 +423,22 @@
</task>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties">
+ <task xsi:type="configure" id="hive_log4j_parameterize">
+ <summary>Updating the Hive Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties">
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize">
+ <summary>Updating the Hive llap Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties">
+ <task xsi:type="configure" id="webhcat_log4j_parameterize">
+ <summary>Updating the Webhcat Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- HBASE -->
<execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for Hbase Master">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/>
@@ -484,6 +500,18 @@
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
</execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties">
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize">
+ <summary>Updating the Storm worker Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties">
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize">
+ <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+
<!-- KAFKA -->
<execute-stage service="KAFKA" component="KAFKA_BROKER" title="Apply config changes for Kafka Broker">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
index 5808aa3..065c265 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
@@ -424,6 +424,22 @@
</task>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties">
+ <task xsi:type="configure" id="hive_log4j_parameterize">
+ <summary>Updating the Hive Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties">
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize">
+ <summary>Updating the Hive llap Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties">
+ <task xsi:type="configure" id="webhcat_log4j_parameterize">
+ <summary>Updating the Webhcat Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- HBASE -->
<execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for Hbase Master">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/>
@@ -479,6 +495,17 @@
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
</execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties">
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize">
+ <summary>Updating the Storm worker Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties">
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize">
+ <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- KAFKA -->
<execute-stage service="KAFKA" component="KAFKA_BROKER" title="Apply config changes for Kafka Broker">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index 9b159a4..774448d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -875,7 +875,8 @@
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HiveEnvClasspathAction">
<summary>Update hive-env content</summary>
</task>
-
+ <task xsi:type="configure" id="hive_log4j_parameterize" />
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade/>
@@ -886,6 +887,10 @@
</component>
<component name="WEBHCAT_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="webhcat_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade/>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -1052,6 +1057,8 @@
<task xsi:type="configure" id="hdp_2_5_0_0_add_storm_security_configs" />
<task xsi:type="configure" id="hdp_2_4_0_0_remove_empty_storm_topology_submission_notifier_plugin_class"/>
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize" />
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
index 2ac82ce..0a0c757 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
@@ -880,6 +880,8 @@
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HiveEnvClasspathAction">
<summary>Update hive-env content</summary>
</task>
+ <task xsi:type="configure" id="hive_log4j_parameterize" />
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize" />
</pre-upgrade>
@@ -891,6 +893,10 @@
</component>
<component name="WEBHCAT_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="webhcat_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade/>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -1047,6 +1053,8 @@
<task xsi:type="configure" id="hdp_2_5_0_0_add_storm_security_configs" />
<task xsi:type="configure" id="hdp_2_4_0_0_remove_empty_storm_topology_submission_notifier_plugin_class"/>
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize" />
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 73e3c38..ddffc79 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -41,6 +41,28 @@
if-type="storm-site"
if-value="15000" />
</definition>
+ <definition xsi:type="configure" id="storm_worker_log4j_parameterize" summary="Parameterizing Storm Worker Log4J Properties">
+ <type>storm-worker-log4j</type>
+ <set key="storm_wrkr_a1_maxfilesize" value="100"/>
+ <set key="storm_wrkr_a1_maxbackupindex" value="9"/>
+ <set key="storm_wrkr_out_maxfilesize" value="100"/>
+ <set key="storm_wrkr_out_maxbackupindex" value="4"/>
+ <set key="storm_wrkr_err_maxfilesize" value="100"/>
+ <set key="storm_wrkr_err_maxbackupindex" value="4"/>
+ <regex-replace key="content" find="A1"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="A1"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/>
+ <regex-replace key="content" find="STDOUT"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="STDOUT"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/>
+ <regex-replace key="content" find="STDERR"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="STDERR"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/>
+ </definition>
+ <definition xsi:type="configure" id="storm_cluster_log4j_parameterize" summary="Parameterizing Storm Cluster Log4J Properties">
+ <type>storm-cluster-log4j</type>
+ <set key="storm_a1_maxfilesize" value="100"/>
+ <set key="storm_a1_maxbackupindex" value="9"/>
+ <regex-replace key="content" find="A1" immediateFlush="false"
 fileName="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
 filePattern="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="A1" immediateFlush="false"
 fileName="${sys:storm.log.dir}/${sys:logfile.name}"
 filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -267,6 +289,20 @@
<type>tez-site</type>
<insert key="tez.task.launch.cmd-opts" value="{{heap_dump_opts}}" insert-type="append" newline-before="false" newline-after="false" />
</definition>
+ <definition xsi:type="configure" id="hive_log4j_parameterize" summary="Parameterizing Hive Log4J Properties">
+ <type>hive-log4j</type>
+ <set key="hive_log_maxfilesize" value="256"/>
+ <set key = "hive_log_maxbackupindex" value="30"/>
+ <regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/>
+ <replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/>
+ </definition>
+ <definition xsi:type="configure" id="hive_llap_log4j_parameterize" summary="Parameterizing Hive llap Log4J Properties">
+ <type>llap-daemon-log4j</type>
+ <set key="hive_llap_log_maxfilesize" value="256"/>
+ <set key = "hive_llap_log_maxbackupindex" value="240"/>
+ <regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/>
+ <regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
<component name="HIVE_SERVER_INTERACTIVE">
@@ -281,6 +317,17 @@
</definition>
</changes>
</component>
+ <component name = "WEBHCAT_SERVER">
+ <changes>
+ <definition xsi:type="configure" id="webhcat_log4j_parameterize" summary="Parameterizing Webhcat Log4J Properties">
+ <type>webhcat-log4j</type>
+ <set key="webhcat_log_maxfilesize" value="256"/>
+ <set key = "webhcat_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxFileSize = {{webhcat_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.MaxBackupIndex = {{webhcat_log_maxbackupindex}}"/>
+ </definition>
+ </changes>
+ </component>
</service>
</services>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index bce539d..abb5428 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -397,7 +397,35 @@
<execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
</execute-stage>
-
+
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm worker Log4J Properties">
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize">
+ <summary>Updating the Storm worker Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Parameterizing Storm cluster Log4J Properties">
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize">
+ <summary>Updating the Storm cluster Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <!--HIVE-->
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive Log4J Properties">
+ <task xsi:type="configure" id="hive_log4j_parameterize">
+ <summary>Updating the Hive Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Parameterizing Hive LLAP Log4J Properties">
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize">
+ <summary>Updating the Hive llap Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties">
+ <task xsi:type="configure" id="webhcat_log4j_parameterize">
+ <summary>Updating the Webhcat Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HIVE" component="HIVE_SERVER" title="Appending heap dump options for Hive">
<task xsi:type="configure" id="hdp_2_6_0_0_hive_append_heap_dump_options"/>
</execute-stage>
http://git-wip-us.apache.org/repos/asf/ambari/blob/9f8401a4/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index 7f9e986..790e50c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -767,6 +767,8 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_6_0_0_tez_append_heap_dump_options"/>
<task xsi:type="configure" id="hdp_2_6_0_0_hive_append_heap_dump_options"/>
+ <task xsi:type="configure" id="hive_log4j_parameterize" />
+ <task xsi:type="configure" id="hive_llap_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade />
@@ -780,6 +782,7 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_6_0_0_hive_llap_append_java_heap_dump_options"/>
<task xsi:type="configure" id="hdp_2_6_0_0_hive_llap_append_heap_dump_options"/>
+ <task xsi:type="configure" id="webhcat_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade />
@@ -927,6 +930,11 @@
<service name="STORM">
<component name="NIMBUS">
+ <pre-upgrade>
+ <task xsi:type="configure" id="storm_worker_log4j_parameterize" />
+ <task xsi:type="configure" id="storm_cluster_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade/>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>