You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mr...@apache.org on 2017/04/28 23:22:39 UTC
ambari git commit: AMBARI-20880. Storm Log4J Regex Replacement on
Upgrade Fails (mradhakrishnan)
Repository: ambari
Updated Branches:
refs/heads/trunk 79cca1c71 -> d51655a83
AMBARI-20880. Storm Log4J Regex Replacement on Upgrade Fails (mradhakrishnan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d51655a8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d51655a8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d51655a8
Branch: refs/heads/trunk
Commit: d51655a83d6c6dd15864e7c4268f8ec9c002cf68
Parents: 79cca1c
Author: Madhuvanthi Radhakrishnan <mr...@hortonworks.com>
Authored: Fri Apr 28 16:21:13 2017 -0700
Committer: Madhuvanthi Radhakrishnan <mr...@hortonworks.com>
Committed: Fri Apr 28 16:21:13 2017 -0700
----------------------------------------------------------------------
.../upgrade/ConfigUpgradeChangeDefinition.java | 3 +--
.../stacks/HDP/2.3/upgrades/config-upgrade.xml | 24 +++++++++++--------
.../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 5 ++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 5 ++++
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 1 +
.../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 1 +
.../stacks/HDP/2.4/upgrades/config-upgrade.xml | 25 +++++++++++---------
.../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 5 ++++
.../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 5 ++++
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 1 +
.../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 1 +
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 19 +++++----------
.../HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml | 6 -----
.../stacks/HDP/2.5/upgrades/upgrade-2.5.xml | 1 -
14 files changed, 59 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
index 5c0fba7..9d65f26 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
@@ -247,8 +247,7 @@ public class ConfigUpgradeChangeDefinition {
}
}catch(Exception e){
- String message = "";
- message = "ConfigUpgradeChangeDefinition: getRegexReplacements : Error while fetching config properties ";
+ String message = "getRegexReplacements : Error while fetching config properties : key - " + regexReplaceObj.key + " find - " + regexReplaceObj.find;
LOG.error(message, e);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index ffa8e61..8b5c07d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -642,22 +642,26 @@
<set key="storm_wrkr_out_maxbackupindex" value="4"/>
<set key="storm_wrkr_err_maxfilesize" value="100"/>
<set key="storm_wrkr_err_maxbackupindex" value="4"/>
- <regex-replace key="content" find="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
- replace-with="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"/>
- <regex-replace key="content" find="A1"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
- replace-with="A1"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/>
- <regex-replace key="content" find="STDOUT"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
- replace-with="STDOUT"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/>
- <regex-replace key="content" find="STDERR"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
- replace-with="STDERR"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/>
+ <regex-replace key="content" find="}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/>
+ <regex-replace key="content" find="}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/>
+ <regex-replace key="content" find="}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/>
</definition>
<definition xsi:type="configure" id="storm_cluster_log4j_parameterize" summary="Parameterizing Storm Cluster Log4J Properties">
<type>storm-cluster-log4j</type>
<set key="storm_a1_maxfilesize" value="100"/>
<set key="storm_a1_maxbackupindex" value="9"/>
- <regex-replace key="content" find="A1" immediateFlush="false"
 fileName="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
 filePattern="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
- replace-with="A1" immediateFlush="false"
 fileName="${sys:storm.log.dir}/${sys:logfile.name}"
 filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/>
+ <regex-replace key="content" find="A1"
 fileName="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
 filePattern="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}.%i">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="A1"
 fileName="${sys:storm.log.dir}/${sys:logfile.name}"
 filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/>
</definition>
+ <definition xsi:type="configure" id="storm_worker_log4j_directory" summary="Update Storm log directory">
+ <type>storm-worker-log4j</type>
+ <replace key="content" find="${sys:storm.log.dir}/${sys:logfile.name}"
+ replace-with="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"/>
+ </definition>
+
</changes>
</component>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index 44367dc..47a7094 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -568,6 +568,11 @@
<summary>Updating the Storm cluster Log4J properties to include parameterizations</summary>
</task>
</execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Update Storm log directory">
+ <task xsi:type="configure" id="storm_worker_log4j_directory">
+ <summary>Update Storm log directory in storm worker log4j</summary>
+ </task>
+ </execute-stage>
<!--ATLAS-->
<execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties">
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
index 9362ae6..18c0c15 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
@@ -666,6 +666,11 @@
<summary>Updating the Storm cluster Log4J properties to include parameterizations</summary>
</task>
</execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Update Storm log directory">
+ <task xsi:type="configure" id="storm_worker_log4j_directory">
+ <summary>Update Storm log directory in storm worker log4j</summary>
+ </task>
+ </execute-stage>
<!--ATLAS-->
<execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties">
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 6981955..f7192ae 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -1092,6 +1092,7 @@
<task xsi:type="configure" id="hdp_2_5_0_0_add_storm_security_configs" />
<task xsi:type="configure" id="storm_worker_log4j_parameterize" />
<task xsi:type="configure" id="storm_cluster_log4j_parameterize" />
+ <task xsi:type="configure" id="storm_worker_log4j_directory" />
</pre-upgrade>
<pre-downgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
index 0440dd3..d98bb53 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
@@ -1128,6 +1128,7 @@
<task xsi:type="configure" id="hdp_2_5_0_0_add_storm_security_configs" />
<task xsi:type="configure" id="storm_worker_log4j_parameterize" />
<task xsi:type="configure" id="storm_cluster_log4j_parameterize" />
+ <task xsi:type="configure" id="storm_worker_log4j_directory" />
</pre-upgrade>
<pre-downgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index f99e764..b3d19d4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -464,21 +464,24 @@
<set key="storm_wrkr_out_maxbackupindex" value="4"/>
<set key="storm_wrkr_err_maxfilesize" value="100"/>
<set key="storm_wrkr_err_maxbackupindex" value="4"/>
- <regex-replace key="content" find="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
- replace-with="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"/>
- <regex-replace key="content" find="A1"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
- replace-with="A1"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/>
- <regex-replace key="content" find="STDOUT"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
- replace-with="STDOUT"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/>
- <regex-replace key="content" find="STDERR"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
- replace-with="STDERR"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/>
- </definition>
+ <regex-replace key="content" find="}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/>
+ <regex-replace key="content" find="}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/>
+ <regex-replace key="content" find="}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/>
+ </definition>
<definition xsi:type="configure" id="storm_cluster_log4j_parameterize" summary="Parameterizing Storm Cluster Log4J Properties">
<type>storm-cluster-log4j</type>
<set key="storm_a1_maxfilesize" value="100"/>
<set key="storm_a1_maxbackupindex" value="9"/>
- <regex-replace key="content" find="A1" immediateFlush="false"
 fileName="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
 filePattern="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
- replace-with="A1" immediateFlush="false"
 fileName="${sys:storm.log.dir}/${sys:logfile.name}"
 filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/>
+ <regex-replace key="content" find="A1"
 fileName="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
 filePattern="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}.%i">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="A1"
 fileName="${sys:storm.log.dir}/${sys:logfile.name}"
 filePattern="${sys:storm.log.dir}/${sys:logfile.name}.%i">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/>
+ </definition>
+ <definition xsi:type="configure" id="storm_worker_log4j_directory" summary="Update Storm log directory">
+ <type>storm-worker-log4j</type>
+ <replace key="content" find="${sys:storm.log.dir}/${sys:logfile.name}"
+ replace-with="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"/>
</definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
index 3a42876..358fb6e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
@@ -495,6 +495,11 @@
<summary>Updating the Storm cluster Log4J properties to include parameterizations</summary>
</task>
</execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Update Storm log directory">
+ <task xsi:type="configure" id="storm_worker_log4j_directory">
+ <summary>Update Storm log directory in storm worker log4j</summary>
+ </task>
+ </execute-stage>
<!-- KAFKA -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
index b812547..368bbe2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
@@ -586,6 +586,11 @@
<summary>Updating the Storm cluster Log4J properties to include parameterizations</summary>
</task>
</execute-stage>
+ <execute-stage service="STORM" component="NIMBUS" title="Update Storm log directory">
+ <task xsi:type="configure" id="storm_worker_log4j_directory">
+ <summary>Update Storm log directory in storm worker log4j</summary>
+ </task>
+ </execute-stage>
<!-- KAFKA -->
<execute-stage service="KAFKA" component="KAFKA_BROKER" title="Apply config changes for Kafka Broker">
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index 5034627..ad9ae33 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -1045,6 +1045,7 @@
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
<task xsi:type="configure" id="storm_worker_log4j_parameterize" />
<task xsi:type="configure" id="storm_cluster_log4j_parameterize" />
+ <task xsi:type="configure" id="storm_worker_log4j_directory" />
</pre-upgrade>
<pre-downgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
index fbd61ea..1eb9836 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
@@ -1090,6 +1090,7 @@
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
<task xsi:type="configure" id="storm_worker_log4j_parameterize" />
<task xsi:type="configure" id="storm_cluster_log4j_parameterize" />
+ <task xsi:type="configure" id="storm_worker_log4j_directory" />
</pre-upgrade>
<pre-downgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 1c2ceff..eb8d7de 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -41,11 +41,6 @@
if-type="storm-site"
if-value="15000" />
</definition>
- <definition xsi:type="configure" id="storm_worker_log4j_update_path" summary="Updating Storm Worker Log4J Path">
- <type>storm-worker-log4j</type>
- <regex-replace key="content" find="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
- replace-with="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"/>
- </definition>
<definition xsi:type="configure" id="storm_worker_log4j_parameterize" summary="Parameterizing Storm Worker Log4J Properties">
<type>storm-worker-log4j</type>
<set key="storm_wrkr_a1_maxfilesize" value="100"/>
@@ -54,14 +49,12 @@
<set key="storm_wrkr_out_maxbackupindex" value="4"/>
<set key="storm_wrkr_err_maxfilesize" value="100"/>
<set key="storm_wrkr_err_maxbackupindex" value="4"/>
- <regex-replace key="content" find="\$\{sys:storm.log.dir}/\$\{sys:logfile.name}"
- replace-with="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"/>
- <regex-replace key="content" find="A1"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
- replace-with="A1"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/>
- <regex-replace key="content" find="STDOUT"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
- replace-with="STDOUT"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/>
- <regex-replace key="content" find="STDERR"
		fileName="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err"
		filePattern="\$\{sys:workers.artifacts}/\$\{sys:storm.id}/\$\{sys:worker.port}/\$\{sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
- replace-with="STDERR"
		fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err"
		filePattern="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/>
+ <regex-replace key="content" find="}.%i.gz">
 <PatternLayout>
 <pattern>\$\{pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="}.%i.gz">
 <PatternLayout>
 <pattern>${pattern}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/>
+ <regex-replace key="content" find="}.out.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="}.out.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/>
+ <regex-replace key="content" find="}.err.%i.gz">
 <PatternLayout>
 <pattern>\$\{patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="(?:[0-9]+) MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="([0-9]+)"
+ replace-with="}.err.%i.gz">
 <PatternLayout>
 <pattern>${patternNoTime}</pattern>
 </PatternLayout>
 <Policies>
 <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
 </Policies>
 <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/>
</definition>
<definition xsi:type="configure" id="storm_cluster_log4j_parameterize" summary="Parameterizing Storm Cluster Log4J Properties">
<type>storm-cluster-log4j</type>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml
index be8b750..55f3685 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml
@@ -312,12 +312,6 @@
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
</execute-stage>
- <execute-stage service="STORM" component="NIMBUS" title="Updating Storm Worker Log4J Path">
- <task xsi:type="configure" id="storm_worker_log4j_update_path">
- <summary>Updating the Storm worker Log4J path</summary>
- </task>
- </execute-stage>
-
<execute-stage service="ATLAS" component="ATLAS_SERVER" title="Updating Atlas TLS Exclude Protocols">
<task xsi:type="configure" id="hdp_2_5_4_0_atlas_exclude_tls_protocol">
<summary>Updating Atlas TLS Exclude Protocols to exclude TLS v1.2</summary>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d51655a8/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
index 210c4e2..e5bbc19 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml
@@ -887,7 +887,6 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_empty_storm_topology_submission_notifier_plugin_class"/>
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
- <task xsi:type="configure" id="storm_worker_log4j_update_path" />
</pre-upgrade>
<pre-downgrade />