You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2017/01/13 23:59:01 UTC
ambari git commit: AMBARI-19538. Add log rotation settings to Hive,
Druid, Storm (Madhuvanthi Radhakrishnan via smohanty)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 d07c86c45 -> 38f0ee59a
AMBARI-19538. Add log rotation settings to Hive, Druid, Storm (Madhuvanthi Radhakrishnan via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/38f0ee59
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/38f0ee59
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/38f0ee59
Branch: refs/heads/branch-2.5
Commit: 38f0ee59a5f0da2b0a52dc24aa3676f8556ab071
Parents: d07c86c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Jan 13 15:58:45 2017 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Jan 13 15:58:45 2017 -0800
----------------------------------------------------------------------
.../0.9.2/configuration/druid-logrotate.xml | 24 ++++++-
.../DRUID/0.9.2/package/scripts/params.py | 3 +
.../0.12.0.2.0/configuration/hive-log4j.xml | 25 ++++++-
.../0.12.0.2.0/configuration/webhcat-log4j.xml | 23 ++++++
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 2 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 11 +++
.../HIVE/0.12.0.2.0/package/scripts/webhcat.py | 2 +-
.../configuration/storm-cluster-log4j.xml | 25 ++++++-
.../0.10.0/configuration/storm-worker-log4j.xml | 75 ++++++++++++++++++--
.../STORM/0.9.1/package/scripts/params_linux.py | 11 +++
.../1.0.1/configuration/storm-cluster-log4j.xml | 25 ++++++-
.../1.0.1/configuration/storm-worker-log4j.xml | 75 ++++++++++++++++++--
.../HIVE/configuration/llap-daemon-log4j.xml | 25 ++++++-
.../stacks/2.0.6/HIVE/test_hive_client.py | 4 +-
.../stacks/2.0.6/HIVE/test_hive_metastore.py | 6 +-
.../stacks/2.0.6/HIVE/test_hive_server.py | 4 +-
.../stacks/2.0.6/HIVE/test_webhcat_server.py | 7 +-
17 files changed, 315 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml
index f612766..b6c9142 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/configuration/druid-logrotate.xml
@@ -21,13 +21,34 @@
-->
<configuration supports_adding_forbidden="true">
<property>
+ <name>druid_log_maxfilesize</name>
+ <value>256</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Druid Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>druid_log_maxbackupindex</name>
+ <value>7</value>
+ <description>The number of backup files</description>
+ <display-name>Druid Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>content</name>
<display-name>druid logrotate template</display-name>
<description>Custom logrotate file</description>
<value><![CDATA[
{{druid_log_dir}}/*.log {
copytruncate
- rotate 7
+ rotate {{druid_log_maxbackupindex}}
daily
nocompress
missingok
@@ -35,6 +56,7 @@
create 660 druid users
dateext
dateformat -%Y-%m-%d-%s
+ size {{druid_log_maxfilesize}}MB
}
]]></value>
<value-attributes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
index c83b16e..032b019 100644
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
@@ -86,6 +86,9 @@ log4j_props = config['configurations']['druid-log4j']['content']
druid_log_level = config['configurations']['druid-log4j']['druid_log_level']
metamx_log_level = config['configurations']['druid-log4j']['metamx_log_level']
root_log_level = config['configurations']['druid-log4j']['root_log_level']
+
+druid_log_maxbackupindex = default('/configurations/druid-logrotate/druid_log_maxbackupindex', 7)
+druid_log_maxfilesize = default('/configurations/druid-logrotate/druid_log_maxfilesize', 256)
logrotate_props = config['configurations']['druid-logrotate']['content']
# Metadata storage
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-log4j.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-log4j.xml
index cc837bb..3473b8c 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-log4j.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-log4j.xml
@@ -21,6 +21,27 @@
-->
<configuration supports_final="false" supports_adding_forbidden="true">
<property>
+ <name>hive_log_maxfilesize</name>
+ <value>256</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Hive Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive_log_maxbackupindex</name>
+ <value>30</value>
+ <description>The number of backup files</description>
+ <display-name>Hive Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>content</name>
<display-name>hive-log4j template</display-name>
<description>Custom log4j.properties</description>
@@ -69,9 +90,11 @@ log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
+#log4j.appender.DRFA.MaxBackupIndex= {{hive_log_maxbackupindex}}
+log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB
log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
# Pattern format: Date LogLevel LoggerName LogMessage
#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
# Debugging Pattern format
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-log4j.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-log4j.xml
index 96562df..4ec22770 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-log4j.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-log4j.xml
@@ -21,6 +21,27 @@
-->
<configuration supports_final="false" supports_adding_forbidden="true">
<property>
+ <name>webhcat_log_maxfilesize</name>
+ <value>256</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Webhcat Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>webhcat_log_maxbackupindex</name>
+ <value>20</value>
+ <description>The number of backup files</description>
+ <display-name>Webhcat Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>content</name>
<display-name>webhcat-log4j template</display-name>
<description>Custom webhcat-log4j.properties</description>
@@ -54,6 +75,8 @@ log4j.threshhold = DEBUG
log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
log4j.appender.standard.File = ${webhcat.log.dir}/${webhcat.log.file}
+log4j.appender.standard.MaxFileSize = {{webhcat_log_maxfilesize}}MB
+log4j.appender.standard.MaxBackupIndex = {{webhcat_log_maxbackupindex}}
# Rollver at midnight
log4j.appender.DRFA.DatePattern = .yyyy-MM-dd
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index 58f01e6..4ac3f8e 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -426,7 +426,7 @@ def fill_conf_dir(component_conf_dir):
mode=mode_identified_for_file,
group=params.user_group,
owner=params.hive_user,
- content=params.log4j_props
+ content=InlineTemplate(params.log4j_props)
)
elif (os.path.exists("{component_conf_dir}/{log4j_filename}.template")):
File(format("{component_conf_dir}/{log4j_filename}"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index a515b62..c55287e 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -441,6 +441,12 @@ hcat_pid_dir = status_params.hcat_pid_dir
hcat_log_dir = config['configurations']['hive-env']['hcat_log_dir']
hcat_env_sh_template = config['configurations']['hcat-env']['content']
+#Hive log4j properties
+webhcat_log_maxfilesize = default("/configurations/webhcat-log4j/webhcat_log_maxfilesize", 256)
+webhcat_log_maxbackupindex = default("/configurations/webhcat-log4j/webhcat_log_maxbackupindex", 20)
+hive_log_maxfilesize = default("/configurations/hive-log4j/hive_log_maxfilesize", 256)
+hive_log_maxbackupindex = default("/configurations/hive-log4j/hive_log_maxbackupindex", 30)
+
#hive-log4j.properties.template
if (('hive-log4j' in config['configurations']) and ('content' in config['configurations']['hive-log4j'])):
log4j_props = config['configurations']['hive-log4j']['content']
@@ -582,6 +588,11 @@ HdfsResource = functools.partial(
# Hive Interactive related
hive_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', [])
has_hive_interactive = len(hive_interactive_hosts) > 0
+
+#llap log4j properties
+hive_llap_log_maxfilesize = default('/configurations/llap-daemon-log4j/hive_llap_log_maxfilesize', 256)
+hive_llap_log_maxbackupindex = default('/configurations/llap-daemon-log4j/hive_llap_log_maxbackupindex', 240)
+
if has_hive_interactive:
llap_daemon_log4j = config['configurations']['llap-daemon-log4j']['content']
llap_cli_log4j2 = config['configurations']['llap-cli-log4j2']['content']
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py
index aa116d6..b519b4e 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/webhcat.py
@@ -128,7 +128,7 @@ def webhcat():
mode=0644,
group=params.user_group,
owner=params.webhcat_user,
- content=params.log4j_webhcat_props
+ content=InlineTemplate(params.log4j_webhcat_props)
)
elif (os.path.exists("{config_dir}/{log4j_webhcat_filename}.template")):
File(format("{config_dir}/{log4j_webhcat_filename}"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-cluster-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-cluster-log4j.xml b/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-cluster-log4j.xml
index 58e1791..92d6c38 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-cluster-log4j.xml
+++ b/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-cluster-log4j.xml
@@ -21,6 +21,27 @@
-->
<configuration supports_final="false" supports_adding_forbidden="true">
<property>
+ <name>storm_a1_maxfilesize</name>
+ <value>100</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Storm Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_a1_maxbackupindex</name>
+ <value>9</value>
+ <description>The number of backup files</description>
+ <display-name>Storm Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>content</name>
<display-name>storm-cluster-log4j template</display-name>
<description>Custom cluster.xml</description>
@@ -55,9 +76,9 @@
<pattern>${pattern}</pattern>
</PatternLayout>
<Policies>
- <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB -->
+ <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
</Policies>
- <DefaultRolloverStrategy max="9"/>
+ <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/>
</RollingFile>
<RollingFile name="ACCESS"
fileName="${sys:storm.log.dir}/access.log"
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-worker-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-worker-log4j.xml b/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-worker-log4j.xml
index adf57fa..45bf289 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-worker-log4j.xml
+++ b/ambari-server/src/main/resources/common-services/STORM/0.10.0/configuration/storm-worker-log4j.xml
@@ -21,6 +21,69 @@
-->
<configuration supports_final="false" supports_adding_forbidden="true">
<property>
+ <name>storm_wrkr_a1_maxfilesize</name>
+ <value>100</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Storm Worker Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_wrkr_a1_maxbackupindex</name>
+ <value>9</value>
+ <description>The number of backup files</description>
+ <display-name>Storm Worker Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_wrkr_out_maxfilesize</name>
+ <value>100</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Storm Worker Standard out Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_wrkr_out_maxbackupindex</name>
+ <value>4</value>
+ <description>The number of backup files</description>
+ <display-name>Storm Worker Standard out Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_wrkr_err_maxfilesize</name>
+ <value>100</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Storm Worker Standard Error Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_wrkr_err_maxbackupindex</name>
+ <value>4</value>
+ <description>The number of backup files</description>
+ <display-name>Storm Worker Standard Error Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>content</name>
<display-name>storm-worker-log4j template</display-name>
<description>Custom worker.xml</description>
@@ -55,9 +118,9 @@
<pattern>${pattern}</pattern>
</PatternLayout>
<Policies>
- <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB -->
+ <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
</Policies>
- <DefaultRolloverStrategy max="9"/>
+ <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/>
</RollingFile>
<RollingFile name="STDOUT"
fileName="${sys:storm.log.dir}/${sys:logfile.name}.out"
@@ -66,9 +129,9 @@
<pattern>${patternNoTime}</pattern>
</PatternLayout>
<Policies>
- <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB -->
+ <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
</Policies>
- <DefaultRolloverStrategy max="4"/>
+ <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/>
</RollingFile>
<RollingFile name="STDERR"
fileName="${sys:storm.log.dir}/${sys:logfile.name}.err"
@@ -77,9 +140,9 @@
<pattern>${patternNoTime}</pattern>
</PatternLayout>
<Policies>
- <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB -->
+ <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
</Policies>
- <DefaultRolloverStrategy max="4"/>
+ <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/>
</RollingFile>
<Syslog name="syslog" format="RFC5424" host="localhost" port="514"
protocol="UDP" appName="[${sys:storm.id}:${sys:worker.port}]" mdcId="mdc" includeMDC="true"
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
index c167d7c..dbb26f6 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/scripts/params_linux.py
@@ -252,6 +252,17 @@ repo_config_username = config['configurations']['ranger-storm-plugin-properties'
ranger_env = config['configurations']['ranger-env']
ranger_plugin_properties = config['configurations']['ranger-storm-plugin-properties']
policy_user = storm_user
+
+#Storm log4j properties
+storm_a1_maxfilesize = default('/configurations/storm-cluster-log4j/storm_a1_maxfilesize', 100)
+storm_a1_maxbackupindex = default('/configurations/storm-cluster-log4j/storm_a1_maxbackupindex', 9)
+storm_wrkr_a1_maxfilesize = default('/configurations/storm-worker-log4j/storm_wrkr_a1_maxfilesize', 100)
+storm_wrkr_a1_maxbackupindex = default('/configurations/storm-worker-log4j/storm_wrkr_a1_maxbackupindex', 9)
+storm_wrkr_out_maxfilesize = default('/configurations/storm-worker-log4j/storm_wrkr_out_maxfilesize', 100)
+storm_wrkr_out_maxbackupindex = default('/configurations/storm-worker-log4j/storm_wrkr_out_maxbackupindex', 4)
+storm_wrkr_err_maxfilesize = default('/configurations/storm-worker-log4j/storm_wrkr_err_maxfilesize', 100)
+storm_wrkr_err_maxbackupindex = default('/configurations/storm-worker-log4j/storm_wrkr_err_maxbackupindex', 4)
+
storm_cluster_log4j_content = config['configurations']['storm-cluster-log4j']['content']
storm_worker_log4j_content = config['configurations']['storm-worker-log4j']['content']
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-cluster-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-cluster-log4j.xml b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-cluster-log4j.xml
index 1709f0d..1cb17f0 100644
--- a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-cluster-log4j.xml
+++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-cluster-log4j.xml
@@ -21,6 +21,27 @@
-->
<configuration supports_final="false" supports_adding_forbidden="true">
<property>
+ <name>storm_a1_maxfilesize</name>
+ <value>100</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Storm Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_a1_maxbackupindex</name>
+ <value>9</value>
+ <description>The number of backup files</description>
+ <display-name>Storm Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>content</name>
<display-name>storm-cluster-log4j template</display-name>
<description>Custom cluster.xml</description>
@@ -53,9 +74,9 @@
<pattern>${pattern}</pattern>
</PatternLayout>
<Policies>
- <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB -->
+ <SizeBasedTriggeringPolicy size="{{storm_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
</Policies>
- <DefaultRolloverStrategy max="9"/>
+ <DefaultRolloverStrategy max="{{storm_a1_maxbackupindex}}"/>
</RollingFile>
<RollingFile name="WEB-ACCESS" immediateFlush="false"
fileName="${sys:storm.log.dir}/access-web-${sys:daemon.name}.log"
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-worker-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-worker-log4j.xml b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-worker-log4j.xml
index a1029a2..c3de7cd 100644
--- a/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-worker-log4j.xml
+++ b/ambari-server/src/main/resources/common-services/STORM/1.0.1/configuration/storm-worker-log4j.xml
@@ -21,6 +21,69 @@
-->
<configuration supports_final="false" supports_adding_forbidden="true">
<property>
+ <name>storm_wrkr_a1_maxfilesize</name>
+ <value>100</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Storm Worker Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_wrkr_a1_maxbackupindex</name>
+ <value>9</value>
+ <description>The number of backup files</description>
+ <display-name>Storm Worker Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_wrkr_out_maxfilesize</name>
+ <value>100</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Storm Worker Standard out Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_wrkr_out_maxbackupindex</name>
+ <value>4</value>
+ <description>The number of backup files</description>
+ <display-name>Storm Worker Standard out Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_wrkr_err_maxfilesize</name>
+ <value>100</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Storm Worker Standard Error Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>storm_wrkr_err_maxbackupindex</name>
+ <value>4</value>
+ <description>The number of backup files</description>
+ <display-name>Storm Worker Standard Error Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>content</name>
<display-name>storm-worker-log4j template</display-name>
<description>Custom worker.xml</description>
@@ -56,9 +119,9 @@
<pattern>${pattern}</pattern>
</PatternLayout>
<Policies>
- <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB -->
+ <SizeBasedTriggeringPolicy size="{{storm_wrkr_a1_maxfilesize}} MB"/> <!-- Or every 100 MB -->
</Policies>
- <DefaultRolloverStrategy max="9"/>
+ <DefaultRolloverStrategy max="{{storm_wrkr_a1_maxbackupindex}}"/>
</RollingFile>
<RollingFile name="STDOUT"
fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.out"
@@ -67,9 +130,9 @@
<pattern>${patternNoTime}</pattern>
</PatternLayout>
<Policies>
- <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB -->
+ <SizeBasedTriggeringPolicy size="{{storm_wrkr_out_maxfilesize}} MB"/> <!-- Or every 100 MB -->
</Policies>
- <DefaultRolloverStrategy max="4"/>
+ <DefaultRolloverStrategy max="{{storm_wrkr_out_maxbackupindex}}"/>
</RollingFile>
<RollingFile name="STDERR"
fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.err"
@@ -78,9 +141,9 @@
<pattern>${patternNoTime}</pattern>
</PatternLayout>
<Policies>
- <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB -->
+ <SizeBasedTriggeringPolicy size="{{storm_wrkr_err_maxfilesize}} MB"/> <!-- Or every 100 MB -->
</Policies>
- <DefaultRolloverStrategy max="4"/>
+ <DefaultRolloverStrategy max="{{storm_wrkr_err_maxbackupindex}}"/>
</RollingFile>
<RollingFile name="METRICS"
fileName="${sys:workers.artifacts}/${sys:storm.id}/${sys:worker.port}/${sys:logfile.name}.metrics"
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml
index 6953530..4790f0f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml
@@ -21,6 +21,27 @@
-->
<configuration supports_final="false" supports_adding_forbidden="true">
<property>
+ <name>hive_llap_log_maxfilesize</name>
+ <value>256</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>Hive LLAP Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive_llap_log_maxbackupindex</name>
+ <value>240</value>
+ <description>The number of backup files</description>
+ <display-name>Hive LLAP Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>content</name>
<display-name>llap-deamon-log4j template</display-name>
<description>Custom llap-daemon-log4j2.properties</description>
@@ -58,8 +79,8 @@ property.llap.daemon.root.logger = console
property.llap.daemon.log.dir = .
property.llap.daemon.log.file = llapdaemon.log
property.llap.daemon.historylog.file = llapdaemon_history.log
-property.llap.daemon.log.maxfilesize = 256MB
-property.llap.daemon.log.maxbackupindex = 240
+property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB
+property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}
# list of all appenders
appenders = console, RFA, HISTORYAPPENDER, query-routing
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
index 2653254..d102b9d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_client.py
@@ -67,7 +67,7 @@ class TestHiveClient(RMFTestCase):
mode = 0644,
)
self.assertResourceCalled('File', '/usr/hdp/current/hive-client/conf/hive-log4j.properties',
- content = 'log4jproperties\nline2',
+ content = InlineTemplate('log4jproperties\nline2'),
owner = 'hive',
group = 'hadoop',
mode = 0644,
@@ -147,7 +147,7 @@ class TestHiveClient(RMFTestCase):
mode = 0644,
)
self.assertResourceCalled('File', '/usr/hdp/current/hive-client/conf/hive-log4j.properties',
- content = 'log4jproperties\nline2',
+ content = InlineTemplate('log4jproperties\nline2'),
owner = 'hive',
group = 'hadoop',
mode = 0644,
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
index bf9c208..821248e 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
@@ -184,7 +184,7 @@ class TestHiveMetastore(RMFTestCase):
mode = 0644,
)
self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-log4j.properties',
- content = 'log4jproperties\nline2',
+ content = InlineTemplate('log4jproperties\nline2'),
owner = 'hive',
group = 'hadoop',
mode = 0644,
@@ -302,7 +302,7 @@ class TestHiveMetastore(RMFTestCase):
mode = 0644,
)
self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-log4j.properties',
- content = 'log4jproperties\nline2',
+ content = InlineTemplate('log4jproperties\nline2'),
owner = 'hive',
group = 'hadoop',
mode = 0644,
@@ -451,7 +451,7 @@ class TestHiveMetastore(RMFTestCase):
owner = 'hive')
self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-log4j.properties',
- content = 'log4jproperties\nline2',
+ content = InlineTemplate('log4jproperties\nline2'),
mode = 420,
group = 'hadoop',
owner = 'hive')
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index 6e97d15..ec3e8f0 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -447,7 +447,7 @@ class TestHiveServer(RMFTestCase):
mode=0644,
)
self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-log4j.properties',
- content='log4jproperties\nline2',
+ content=InlineTemplate('log4jproperties\nline2'),
owner='hive',
group='hadoop',
mode=0644,
@@ -653,7 +653,7 @@ class TestHiveServer(RMFTestCase):
mode=0644,
)
self.assertResourceCalled('File', '/usr/hdp/current/hive-server2/conf/hive-log4j.properties',
- content='log4jproperties\nline2',
+ content=InlineTemplate('log4jproperties\nline2'),
owner='hive',
group='hadoop',
mode=0644,
http://git-wip-us.apache.org/repos/asf/ambari/blob/38f0ee59/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
index 2b26dd8..f93ea09 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
@@ -22,6 +22,7 @@ from mock.mock import MagicMock, patch
from stacks.utils.RMFTestCase import *
from resource_management.core.exceptions import Fail
+
@patch("os.path.isfile", new = MagicMock(return_value=True))
@patch("glob.glob", new = MagicMock(return_value=["one", "two"]))
class TestWebHCatServer(RMFTestCase):
@@ -227,7 +228,7 @@ class TestWebHCatServer(RMFTestCase):
create_parents = True
)
self.assertResourceCalled('File', '/etc/hive-webhcat/conf/webhcat-log4j.properties',
- content = 'log4jproperties\nline2',
+ content = InlineTemplate('log4jproperties\nline2'),
owner = 'hcat',
group = 'hadoop',
mode = 0644,
@@ -273,7 +274,7 @@ class TestWebHCatServer(RMFTestCase):
create_parents = True
)
self.assertResourceCalled('File', '/etc/hive-webhcat/conf/webhcat-log4j.properties',
- content = 'log4jproperties\nline2',
+ content = InlineTemplate('log4jproperties\nline2'),
owner = 'hcat',
group = 'hadoop',
mode = 0644,
@@ -531,7 +532,7 @@ class TestWebHCatServer(RMFTestCase):
create_parents = True)
self.assertResourceCalled('File', '/usr/hdp/current/hive-webhcat/etc/webhcat/webhcat-log4j.properties',
- content = 'log4jproperties\nline2',
+ content = InlineTemplate('log4jproperties\nline2'),
owner = 'hcat',
group = 'hadoop',
mode = 0644)