You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2017/01/17 19:12:48 UTC
[2/2] ambari git commit: AMBARI-19541. Add log rotation settings -
handle HDP upgrade scenario (Madhuvanthi Radhakrishnan via smohanty)
AMBARI-19541. Add log rotation settings - handle HDP upgrade scenario (Madhuvanthi Radhakrishnan via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/21e8e4bd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/21e8e4bd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/21e8e4bd
Branch: refs/heads/branch-2.5
Commit: 21e8e4bd88fbf87a0c732fc56c289ec790e396a9
Parents: e700484
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Jan 17 10:53:45 2017 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Jan 17 11:08:35 2017 -0800
----------------------------------------------------------------------
.../upgrade/ConfigUpgradeChangeDefinition.java | 110 ++++++++++-
.../state/stack/upgrade/ConfigureTask.java | 9 +-
.../stacks/HDP/2.3/upgrades/config-upgrade.xml | 148 +++++++++++++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 90 +++++++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 85 +++++++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 89 +++++++++
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 49 ++++-
.../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 43 ++++-
.../stacks/HDP/2.4/upgrades/config-upgrade.xml | 153 +++++++++++++++
.../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 90 +++++++++
.../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 84 ++++++++
.../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 39 +++-
.../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 39 +++-
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 190 ++++++++++++++++++-
.../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 95 +++++++++-
.../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 41 ++++
.../src/main/resources/upgrade-config.xsd | 14 +-
17 files changed, 1356 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/21e8e4bd/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
index 5428ea7..31df790 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigUpgradeChangeDefinition.java
@@ -20,6 +20,9 @@ package org.apache.ambari.server.state.stack.upgrade;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -28,6 +31,8 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -130,6 +135,9 @@ public class ConfigUpgradeChangeDefinition {
@XmlElement(name="replace")
private List<Replace> replacements;
+ @XmlElement(name="regex-replace")
+ private List<RegexReplace> regexReplacements;
+
/**
* @return the config type
*/
@@ -198,6 +206,50 @@ public class ConfigUpgradeChangeDefinition {
}
/**
+ * @return the replacement tokens, never {@code null}
+ */
+ public List<Replace> getRegexReplacements(Cluster cluster) {
+
+ if (null == regexReplacements) {
+
+ return Collections.emptyList();
+ }
+
+ List<Replace> list = new ArrayList<>();
+ for (RegexReplace regexReplaceObj : regexReplacements) {
+ if (null == regexReplaceObj.key || null == regexReplaceObj.find || null == regexReplaceObj.replaceWith) {
+ LOG.warn(String.format("Replacement %s is invalid", regexReplaceObj));
+ continue;
+ }
+
+ try{
+ Config config = cluster.getDesiredConfigByType(configType);
+
+ Map<String, String> properties = config.getProperties();
+ String content = properties.get(regexReplaceObj.key);
+
+ Pattern REGEX = Pattern.compile(regexReplaceObj.find, Pattern.MULTILINE);
+
+ Matcher patternMatchObj = REGEX.matcher(content);
+ if (patternMatchObj.find() && patternMatchObj.groupCount()==1) {
+ regexReplaceObj.find = patternMatchObj.group();
+ Replace rep = regexReplaceObj.copyToReplaceObject();
+ list.add(rep);
+ }
+
+ }catch(Exception e){
+ String message = "";
+ message = "ConfigUpgradeChangeDefinition: getRegexReplacements : Error while fetching config properties ";
+ LOG.error(message, e);
+
+ }
+
+ }
+ return list;
+ }
+
+
+ /**
* Used for configuration updates that should mask their values from being
* printed in plain text.
*/
@@ -379,4 +431,60 @@ public class ConfigUpgradeChangeDefinition {
'}';
}
}
-}
\ No newline at end of file
+
+ /**
+ * Used to replace strings in a key with other strings. More complex
+ * scenarios are possible with regex.
+ */
+ @XmlAccessorType(XmlAccessType.FIELD)
+ @XmlType(name = "regex-replace")
+ public static class RegexReplace extends Masked{
+ /**
+ * The key name
+ */
+ @XmlAttribute(name="key")
+ public String key;
+
+ /**
+ * The string to find
+ */
+ @XmlAttribute(name="find")
+ public String find;
+
+ /**
+ * The string to replace
+ */
+ @XmlAttribute(name="replace-with")
+ public String replaceWith;
+
+ @Override
+ public String toString() {
+ return "RegexReplace{" +
+ "key='" + key + '\'' +
+ ", find='" + find + '\'' +
+ ", replaceWith='" + replaceWith + '\'' +
+ ", ifKey='" + ifKey + '\'' +
+ ", ifType='" + ifType + '\'' +
+ ", ifValue='" + ifValue + '\'' +
+ ", ifKeyState='" + ifKeyState + '\'' +
+ '}';
+ }
+
+ /***
+ * Copies a RegexReplace type object to Replace object.
+ * @return Replace object
+ */
+ public Replace copyToReplaceObject(){
+ Replace rep = new Replace();
+ rep.find = this.find;
+ rep.key = this.key;
+ rep.replaceWith = this.replaceWith;
+ rep.ifKey = this.ifKey;
+ rep.ifType = this.ifType;
+ rep.ifValue = this.ifValue;
+ rep.ifKeyState = this.ifKeyState;
+
+ return rep;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/21e8e4bd/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
index d7bb338..f256eb0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
@@ -211,7 +211,12 @@ public class ConfigureTask extends ServerSideActionTask {
}
// replacements
- List<Replace> replacements = definition.getReplacements();
+
+ List<Replace> replacements = new ArrayList<Replace>();
+ replacements.addAll(definition.getReplacements());
+ //Fetch the replacements that used regex to find a string
+ replacements.addAll(definition.getRegexReplacements(cluster));
+
if( null != replacements && !replacements.isEmpty() ){
List<Replace> allowedReplacements = getValidReplacements(cluster, definition.getConfigType(), replacements);
configParameters.put(ConfigureTask.PARAMETER_REPLACEMENTS, m_gson.toJson(allowedReplacements));
@@ -319,4 +324,4 @@ public class ConfigureTask extends ServerSideActionTask {
return config.getProperties().get(propertyKey);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/21e8e4bd/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index a1f03e8..09de45c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -64,6 +64,19 @@
<transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" />
</definition>
+ <!-- HBase Rolling properties for log4j need to be parameterized. -->
+ <definition xsi:type="configure" id="hbase_log4j_parameterize" summary="Parameterizing HBase Log4J Properties">
+ <type>hbase-log4j</type>
+ <set key="hbase_log_maxfilesize" value="256"/>
+ <set key="hbase_log_maxbackupindex" value="20"/>
+ <set key="hbase_security_log_maxfilesize" value="256"/>
+ <set key="hbase_security_log_maxbackupindex" value="20"/>
+ <regex-replace key="content" find="hbase.log.maxfilesize=([0-9]+)MB" replace-with="hbase.log.maxfilesize={{hbase_log_maxfilesize}}MB"/>
+ <regex-replace key="content" find="hbase.log.maxbackupindex=([0-9]+)" replace-with="hbase.log.maxbackupindex={{hbase_log_maxbackupindex}}"/>
+ <regex-replace key="content" find="hbase.security.log.maxfilesize=([0-9]+)MB" replace-with="hbase.security.log.maxfilesize={{hbase_security_log_maxfilesize}}MB"/>
+ <regex-replace key="content" find="hbase.security.log.maxbackupindex=([0-9]+)" replace-with="hbase.security.log.maxbackupindex={{hbase_security_log_maxbackupindex}}"/>
+ </definition>
+
</changes>
</component>
</service>
@@ -232,9 +245,40 @@
<type>ranger-env</type>
<transfer operation="delete" delete-key="bind_anonymous" />
</definition>
+ <definition xsi:type="configure" id="admin_log4j_parameterize" summary="Parameterizing Ranger Log4J Properties">
+ <type>admin-log4j</type>
+ <set key="ranger_xa_log_maxfilesize" value="256"/>
+ <set key="ranger_xa_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.xa_log_appender.MaxFileSize={{ranger_xa_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.xa_log_appender.MaxBackupIndex={{ranger_xa_log_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
+
+ <component name="RANGER_USERSYNC">
+ <changes>
+ <definition xsi:type="configure" id="usersync_log4j_parameterize" summary="Parameterizing Ranger Usersync Log4J Properties">
+ <type>usersync-log4j</type>
+ <set key="ranger_usersync_log_maxfilesize" value="256"/>
+ <set key="ranger_usersync_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxFileSize = {{ranger_usersync_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxBackupIndex = {{ranger_usersync_log_maxbackupindex}}"/>
+ </definition>
+ </changes>
+ </component>
+
+ <component name="RANGER_TAGSYNC">
+ <changes>
+ <definition xsi:type="configure" id="tagsync_log4j_parameterize" summary="Parameterizing Ranger Tagsync Log4J Properties">
+ <type>tagsync-log4j</type>
+ <set key="ranger_tagsync_log_maxfilesize" value="256"/>
+ <set key="ranger_tagsync_log_number_of_backup_files" value="20"/>
+ <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxFileSize = {{ranger_tagsync_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxBackupIndex = {{ranger_tagsync_log_number_of_backup_files}}"/>
+ </definition>
+ </changes>
+ </component>
</service>
<service name="RANGER_KMS">
@@ -250,6 +294,17 @@
<transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" />
<transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" />
</definition>
+ <definition xsi:type="configure" id="kms_log4j_parameterize" summary="Parameterizing Ranger KMS Log4J Properties">
+ <type>kms-log4j</type>
+ <set key="ranger_kms_log_maxfilesize" value="256"/>
+ <set key="ranger_kms_log_maxbackupindex" value="20"/>
+ <set key="ranger_kms_audit_log_maxfilesize" value="256"/>
+ <set key="ranger_kms_audit_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms.MaxFileSize = {{ranger_kms_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms.MaxBackupIndex = {{ranger_kms_log_maxbackupindex}}"/>
+ <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms-audit.MaxFileSize = {{ranger_kms_audit_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms-audit.MaxBackupIndex = {{ranger_kms_audit_log_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -274,6 +329,19 @@
<transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" />
<transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" />
</definition>
+
+ <!-- HDFS Rolling properties for log4j need to be parameterized. -->
+ <definition xsi:type="configure" id="hdfs_log4j_parameterize" summary="Parameterizing Hdfs Log4J Properties">
+ <type>hdfs-log4j</type>
+ <set key="hadoop_log_max_backup_size" value="256"/>
+ <set key="hadoop_log_number_of_backup_files" value="10"/>
+ <set key="hadoop_security_log_max_backup_size" value="256"/>
+ <set key="hadoop_security_log_number_of_backup_files" value="20"/>
+ <regex-replace key="content" find="log4j.appender.RFA.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RFA.MaxFileSize={{hadoop_log_max_backup_size}}MB"/>
+ <regex-replace key="content" find="log4j.appender.RFA.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RFA.MaxBackupIndex={{hadoop_log_number_of_backup_files}}"/>
+ <regex-replace key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/>
+ <regex-replace key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -339,6 +407,12 @@
<type>oozie-site</type>
<replace key="oozie.services" find="org.apache.oozie.service.CoordinatorStoreService," replace-with="" />
</definition>
+ <!-- Oozie Rolling properties for log4j need to be parameterized. -->
+ <definition xsi:type="configure" id="oozie_log4j_parameterize" summary="Parameterizing Oozie Log4J Properties">
+ <type>oozie-log4j</type>
+ <set key="oozie_log_maxhistory" value="720"/>
+ <regex-replace key="content" find="^log4j.appender.oozie.RollingPolicy.MaxHistory=([0-9]+)" replace-with="log4j.appender.oozie.RollingPolicy.MaxHistory={{oozie_log_maxhistory}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -361,6 +435,17 @@
<set key="inter.broker.protocol.version" value="0.9.0.0" />
<set key="log.message.format.version" value="0.9.0.0" />
</definition>
+ <definition xsi:type="configure" id="kafka_log4j_parameterize" summary="Parameterizing Kafka Log4J Properties">
+ <type>kafka-log4j</type>
+ <set key="kafka_log_maxfilesize" value="256"/>
+ <set key="kafka_log_maxbackupindex" value="20"/>
+ <set key="controller_log_maxfilesize" value="256"/>
+ <set key="controller_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kafkaAppender.MaxFileSize = {{kafka_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kafkaAppender.MaxBackupIndex = {{kafka_log_maxbackupindex}}"/>
+ <replace key="content" find="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.controllerAppender.MaxFileSize = {{controller_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.controllerAppender.MaxBackupIndex = {{controller_log_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -378,6 +463,14 @@
<transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" />
<transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" />
</definition>
+ <!-- Yarn Rolling properties for log4j need to be parameterized. -->
+ <definition xsi:type="configure" id="yarn_log4j_parameterize" summary="Parameterizing Yarn Log4J Properties">
+ <type>yarn-log4j</type>
+ <set key="yarn_rm_summary_log_max_backup_size" value="256"/>
+ <set key="yarn_rm_summary_log_number_of_backup_files" value="20"/>
+ <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/>
+ <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -473,6 +566,20 @@
<transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" />
<transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" />
</definition>
+ <definition xsi:type="configure" id="knox_gateway_log4j_parameterize" summary="Parameterizing Knox Gateway Log4J Properties">
+ <type>gateway-log4j</type>
+ <set key="knox_gateway_log_maxfilesize" value="256"/>
+ <set key="knox_gateway_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxFileSize = {{knox_gateway_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxBackupIndex = {{knox_gateway_log_maxbackupindex}}"/>
+ </definition>
+ <definition xsi:type="configure" id="knox_ldap_log4j_parameterize" summary="Parameterizing Knox Ldap Log4J Properties">
+ <type>ldap-log4j</type>
+ <set key="knox_ldap_log_maxfilesize" value="256"/>
+ <set key="knox_ldap_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxFileSize = {{knox_ldap_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxBackupIndex = {{knox_ldap_log_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -484,8 +591,49 @@
<type>falcon-startup.properties</type>
<set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/>
</definition>
+ <definition xsi:type="configure" id="falcon_log4j_parameterize" summary="Parameterizing Falcon Log4J Properties">
+ <type>falcon-log4j</type>
+ <set key="falcon_log_maxfilesize" value="256"/>
+ <set key="falcon_log_maxbackupindex" value="20"/>
+ <set key="falcon_security_log_maxfilesize" value="256"/>
+ <set key="falcon_security_log_maxbackupindex" value="20"/>
+ <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxFileSize" value="{{falcon_log_maxfilesize}}MB" />"/>
+ <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxBackupIndex" value="{{falcon_log_maxbackupindex}}" />"/>
+ <replace key="content" find="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxFileSize" value="{{falcon_security_log_maxfilesize}}MB"/>"/>
+ <replace key="content" find="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxBackupIndex" value="{{falcon_security_log_maxbackupindex}}"/>"/>
+ </definition>
+ </changes>
+ </component>
+ </service>
+
+ <service name="ATLAS">
+ <component name="ATLAS_SERVER">
+ <changes>
+ <definition xsi:type="configure" id="atlas_log4j_parameterize" summary="Parameterizing Atlas Log4J Properties">
+ <type>atlas-log4j</type>
+ <set key="atlas_log_max_backup_size" value="256"/>
+ <set key="atlas_log_number_of_backup_files" value="20"/>
+ <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">\n<param name="MaxFileSize" value="{{atlas_log_max_backup_size}}MB" />"/>
+ <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">\n<param name="MaxFileSize" value="{{atlas_log_number_of_backup_files}}" />"/>
+ </definition>
+ </changes>
+ </component>
+ </service>
+
+ <service name="ZOOKEEPER">
+ <component name="ZOOKEEPER_SERVER">
+ <changes>
+ <!-- Zookeeper Rolling properties for log4j need to be parameterized. -->
+ <definition xsi:type="configure" id="zookeeper_log4j_parameterize" summary="Parameterizing ZooKeeper Log4J Properties">
+ <type>zookeeper-log4j</type>
+ <set key="zookeeper_log_max_backup_size" value="10"/>
+ <set key="zookeeper_log_number_of_backup_files" value="10"/>
+ <regex-replace key="content" find="^log4j.appender.ROLLINGFILE.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.ROLLINGFILE.MaxFileSize={{zookeeper_log_max_backup_size}}MB"/>
+ <regex-replace key="content" find="^#log4j.appender.ROLLINGFILE.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.ROLLINGFILE.MaxBackupIndex={{zookeeper_log_number_of_backup_files}}"/>
+ </definition>
</changes>
</component>
</service>
</services>
</upgrade-config-changes>
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/21e8e4bd/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
index 8dff078..81c470a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
@@ -258,6 +258,12 @@
<task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/>
</execute-stage>
+ <execute-stage service="HDFS" component="NAMENODE" title="Parameterizing Hdfs Log4J Properties">
+ <task xsi:type="configure" id="hdfs_log4j_parameterize">
+ <summary>Updating the Hdfs Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- YARN -->
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig">
@@ -265,6 +271,13 @@
</task>
</execute-stage>
+ <!--Yarn-->
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Parameterizing Yarn Log4J Properties Resource Manager">
+ <task xsi:type="configure" id="yarn_log4j_parameterize">
+ <summary>Updating the Yarn Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath">
<summary>Verifying LZO codec path for mapreduce</summary>
@@ -277,6 +290,12 @@
<task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/>
</execute-stage>
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Parameterizing HBase Log4J Properties">
+ <task xsi:type="configure" id="hbase_log4j_parameterize">
+ <summary>Updating the Hbase Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction">
<summary>Update HBase Env Configuration</summary>
@@ -310,6 +329,12 @@
</task>
</execute-stage>
+ <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Parameterizing Oozie Log4J Properties">
+ <task xsi:type="configure" id="oozie_log4j_parameterize">
+ <summary>Updating the Oozie Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
<task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
</execute-stage>
@@ -342,6 +367,71 @@
<execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
</execute-stage>
+
+ <!--ZOOKEEPER-->
+ <execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Parameterizing Zookeeper Log4J Properties">
+ <task xsi:type="configure" id="zookeeper_log4j_parameterize">
+ <summary>Updating the Zookeeper Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <!--ATLAS-->
+ <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties">
+ <task xsi:type="configure" id="atlas_log4j_parameterize">
+ <summary>Updating the Atlas Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <!--KAFKA-->
+ <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Parameterizing Kafka Log4J Properties">
+ <task xsi:type="configure" id="kafka_log4j_parameterize">
+ <summary>Updating the Kafka Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <!--RANGER-->
+ <execute-stage service="RANGER" component="RANGER_ADMIN" title="Parameterizing Ranger Admin Log4J Properties">
+ <task xsi:type="configure" id="admin_log4j_parameterize">
+ <summary>Updating the Ranger admin Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="RANGER" component="RANGER_USERSYNC" title="Parameterizing Ranger Usersync Log4J Properties">
+ <task xsi:type="configure" id="usersync_log4j_parameterize">
+ <summary>Updating the Ranger usersync Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="RANGER" component="RANGER_TAGSYNC" title="Parameterizing Ranger Tagsync Log4J Properties">
+ <task xsi:type="configure" id="tagsync_log4j_parameterize">
+ <summary>Updating the Ranger tagsync Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <!--RANGER-KMS-->
+ <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Parameterizing Ranger Kms Log4J Properties">
+ <task xsi:type="configure" id="kms_log4j_parameterize">
+ <summary>Updating the KMS Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <!--KNOX-->
+ <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Gateway Log4J Properties">
+ <task xsi:type="configure" id="knox_gateway_log4j_parameterize">
+ <summary>Updating the Knox Gateway Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Ldap Log4J Properties">
+ <task xsi:type="configure" id="knox_ldap_log4j_parameterize">
+ <summary>Updating the Knox Ldap Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <!--FALCON-->
+ <execute-stage service="FALCON" component="FALCON_SERVER" title="Parameterizing Falcon Log4J Properties">
+ <task xsi:type="configure" id="falcon_log4j_parameterize">
+ <summary>Updating the Falcon Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
</group>
<!--
http://git-wip-us.apache.org/repos/asf/ambari/blob/21e8e4bd/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index 404d162..3e61aad 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -279,6 +279,12 @@
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db"/>
</execute-stage>
+ <execute-stage service="HDFS" component="NAMENODE" title="Parameterizing Hdfs Log4J Properties">
+ <task xsi:type="configure" id="hdfs_log4j_parameterize">
+ <summary>Updating the Hdfs Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- YARN -->
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig">
@@ -290,6 +296,12 @@
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
</execute-stage>
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Parameterizing Yarn Log4J Properties Resource Manager">
+ <task xsi:type="configure" id="yarn_log4j_parameterize">
+ <summary>Updating the Yarn Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
<task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
</execute-stage>
@@ -310,6 +322,12 @@
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/>
</execute-stage>
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Parameterizing HBase Log4J Properties">
+ <task xsi:type="configure" id="hbase_log4j_parameterize">
+ <summary>Updating the Hbase Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction">
<summary>Update HBase Env Configuration</summary>
@@ -374,15 +392,34 @@
<task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
</execute-stage>
+ <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Parameterizing Oozie Log4J Properties">
+ <task xsi:type="configure" id="oozie_log4j_parameterize">
+ <summary>Updating the Oozie Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!--FALCON-->
<execute-stage service="FALCON" component="FALCON_SERVER" title="Apply config changes for Falcon">
<task xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"/>
</execute-stage>
+ <execute-stage service="FALCON" component="FALCON_SERVER" title="Parameterizing Falcon Log4J Properties">
+ <task xsi:type="configure" id="falcon_log4j_parameterize">
+ <summary>Updating the Falcon Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <!--KAFKA-->
<execute-stage service="KAFKA" component="KAFKA_BROKER" title="Apply config changes for Kafka Broker">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db"/>
</execute-stage>
+ <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Parameterizing Kafka Log4J Properties">
+ <task xsi:type="configure" id="kafka_log4j_parameterize">
+ <summary>Updating the Kafka Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- SPARK -->
<execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config changes for Spark JobHistoryServer">
<task xsi:type="configure" id="hdp_2_4_0_0_spark_jobhistoryserver"/>
@@ -424,6 +461,12 @@
<task xsi:type="configure" id="hdp_2_5_0_0_set_external_solrCloud_flag"/>
</execute-stage>
+ <execute-stage service="RANGER" component="RANGER_ADMIN" title="Parameterizing Ranger Admin Log4J Properties">
+ <task xsi:type="configure" id="admin_log4j_parameterize">
+ <summary>Updating the Ranger admin Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="RANGER" component="RANGER_ADMIN" title="Calculating Ranger Properties">
<condition xsi:type="security" type="kerberos"/>
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.RangerKerberosConfigCalculation">
@@ -437,6 +480,17 @@
</task>
</execute-stage>
+ <execute-stage service="RANGER" component="RANGER_USERSYNC" title="Parameterizing Ranger Usersync Log4J Properties">
+ <task xsi:type="configure" id="usersync_log4j_parameterize">
+ <summary>Updating the Ranger usersync Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="RANGER" component="RANGER_TAGSYNC" title="Parameterizing Ranger Tagsync Log4J Properties">
+ <task xsi:type="configure" id="tagsync_log4j_parameterize">
+ <summary>Updating the Ranger tagsync Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- RANGER KMS -->
<execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger KMS Server">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db"/>
@@ -449,11 +503,28 @@
</task>
</execute-stage>
+ <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Parameterizing Ranger Kms Log4J Properties">
+ <task xsi:type="configure" id="kms_log4j_parameterize">
+ <summary>Updating the KMS Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- KNOX -->
<execute-stage service="KNOX" component="KNOX_GATEWAY" title="Apply config changes for Knox Gateway">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_knox_audit_db"/>
</execute-stage>
+ <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Gateway Log4J Properties">
+ <task xsi:type="configure" id="knox_gateway_log4j_parameterize">
+ <summary>Updating the Knox Gateway Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+ <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Ldap Log4J Properties">
+ <task xsi:type="configure" id="knox_ldap_log4j_parameterize">
+ <summary>Updating the Knox Ldap Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- STORM -->
<execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Storm Nimbus">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_storm_audit_db"/>
@@ -485,6 +556,20 @@
<execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
</execute-stage>
+
+ <!--ATLAS-->
+ <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties">
+ <task xsi:type="configure" id="atlas_log4j_parameterize">
+ <summary>Updating the Atlas Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <!--ZOOKEEPER-->
+ <execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Parameterizing Zookeeper Log4J Properties">
+ <task xsi:type="configure" id="zookeeper_log4j_parameterize">
+ <summary>Updating the Zookeeper Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
</group>
<!--
http://git-wip-us.apache.org/repos/asf/ambari/blob/21e8e4bd/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
index 549e5a4..cffecaa3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
@@ -280,6 +280,12 @@
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db"/>
</execute-stage>
+ <execute-stage service="HDFS" component="NAMENODE" title="Parameterizing Hdfs Log4J Properties">
+ <task xsi:type="configure" id="hdfs_log4j_parameterize">
+ <summary>Updating the Hdfs Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- YARN -->
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig">
@@ -295,6 +301,12 @@
<task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
</execute-stage>
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Parameterizing Yarn Log4J Properties Resource Manager">
+ <task xsi:type="configure" id="yarn_log4j_parameterize">
+ <summary>Updating the Yarn Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath">
<summary>Verifying LZO codec path for mapreduce</summary>
@@ -311,6 +323,13 @@
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db"/>
</execute-stage>
+ <!--HBASE-->
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Parameterizing HBase Log4J Properties">
+ <task xsi:type="configure" id="hbase_log4j_parameterize">
+ <summary>Updating the Hbase Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HBASE" component="HBASE_MASTER" title="Update HBase Configuration">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.HBaseEnvMaxDirectMemorySizeAction">
<summary>Update HBase Env Configuration</summary>
@@ -375,15 +394,34 @@
<task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
</execute-stage>
+ <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Parameterizing Oozie Log4J Properties">
+ <task xsi:type="configure" id="oozie_log4j_parameterize">
+ <summary>Updating the Oozie Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!--FALCON-->
<execute-stage service="FALCON" component="FALCON_SERVER" title="Apply config changes for Falcon">
<task xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"/>
</execute-stage>
+ <execute-stage service="FALCON" component="FALCON_SERVER" title="Parameterizing Falcon Log4J Properties">
+ <task xsi:type="configure" id="falcon_log4j_parameterize">
+ <summary>Updating the Falcon Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <!--KAFKA-->
<execute-stage service="KAFKA" component="KAFKA_BROKER" title="Apply config changes for Kafka Broker">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db"/>
</execute-stage>
+ <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Parameterizing Kafka Log4J Properties">
+ <task xsi:type="configure" id="kafka_log4j_parameterize">
+ <summary>Updating the Kafka Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- SPARK -->
<execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config changes for Spark JobHistoryServer">
<task xsi:type="configure" id="hdp_2_4_0_0_spark_jobhistoryserver"/>
@@ -425,6 +463,12 @@
<task xsi:type="configure" id="hdp_2_5_0_0_set_external_solrCloud_flag"/>
</execute-stage>
+ <execute-stage service="RANGER" component="RANGER_ADMIN" title="Parameterizing Ranger Admin Log4J Properties">
+ <task xsi:type="configure" id="admin_log4j_parameterize">
+ <summary>Updating the Ranger admin Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="RANGER" component="RANGER_ADMIN" title="Calculating Ranger Properties">
<condition xsi:type="security" type="kerberos"/>
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.RangerKerberosConfigCalculation">
@@ -441,12 +485,30 @@
<execute-stage service="RANGER" component="RANGER_ADMIN" title="Apply config changes for Ranger Admin">
<task xsi:type="configure" id="hdp_2_6_0_0_remove_bind_anonymous"/>
</execute-stage>
+
+ <execute-stage service="RANGER" component="RANGER_USERSYNC" title="Parameterizing Ranger Usersync Log4J Properties">
+ <task xsi:type="configure" id="usersync_log4j_parameterize">
+ <summary>Updating the Ranger usersync Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <execute-stage service="RANGER" component="RANGER_TAGSYNC" title="Parameterizing Ranger Tagsync Log4J Properties">
+ <task xsi:type="configure" id="tagsync_log4j_parameterize">
+ <summary>Updating the Ranger tagsync Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
<!-- RANGER KMS -->
<execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger KMS Server">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db"/>
</execute-stage>
+ <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config changes for Ranger kms">
+ <task xsi:type="configure" id="kms_log4j_parameterize">
+ <summary>Updating the KMS Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Calculating Ranger Properties">
<condition xsi:type="security" type="kerberos"/>
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.RangerKmsProxyConfig">
@@ -459,6 +521,18 @@
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_knox_audit_db"/>
</execute-stage>
+ <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Gateway Log4J Properties">
+ <task xsi:type="configure" id="knox_gateway_log4j_parameterize">
+ <summary>Updating the Knox Gateway Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <execute-stage service="KNOX" component="KNOX_GATEWAY" title="Parameterizing Knox Ldap Log4J Properties">
+ <task xsi:type="configure" id="knox_ldap_log4j_parameterize">
+ <summary>Updating the Knox Ldap Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
<!-- STORM -->
<execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Storm Nimbus">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_storm_audit_db"/>
@@ -485,6 +559,20 @@
<execute-stage service="STORM" component="NIMBUS" title="Apply config changes for Nimbus">
<task xsi:type="configure" id="increase_storm_zookeeper_timeouts"/>
</execute-stage>
+
+ <!--ATLAS-->
+ <execute-stage service="ATLAS" component="ATLAS_SERVER" title="Parameterizing Atlas Log4J Properties">
+ <task xsi:type="configure" id="atlas_log4j_parameterize">
+ <summary>Updating the Atlas Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
+
+ <!--ZOOKEEPER-->
+ <execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Parameterizing Zookeeper Log4J Properties">
+ <task xsi:type="configure" id="zookeeper_log4j_parameterize">
+ <summary>Updating the Zookeeper Log4J properties to include parameterizations</summary>
+ </task>
+ </execute-stage>
</group>
<!--
@@ -1364,3 +1452,4 @@
</service>
</processing>
</upgrade>
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/21e8e4bd/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 6d44b57..5ae6876 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -470,6 +470,10 @@
<processing>
<service name="ZOOKEEPER">
<component name="ZOOKEEPER_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="zookeeper_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade />
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -509,6 +513,7 @@
<script>scripts/ranger_admin.py</script>
<function>setup_ranger_java_patches</function>
</task>
+ <task xsi:type="configure" id="admin_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade copy-upgrade="true" />
@@ -520,6 +525,20 @@
</component>
<component name="RANGER_USERSYNC">
+ <pre-upgrade>
+ <task xsi:type="configure" id="usersync_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade/>
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+
+ <component name="RANGER_TAGSYNC">
+ <pre-upgrade>
+ <task xsi:type="configure" id="tagsync_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade />
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -529,6 +548,7 @@
<service name="RANGER_KMS">
<component name="RANGER_KMS_SERVER">
<pre-upgrade>
+ <task xsi:type="configure" id="kms_log4j_parameterize" />
<task xsi:type="execute" hosts="any" sequential="true">
<summary>Upgrading Ranger KMS database schema</summary>
<script>scripts/kms_server.py</script>
@@ -555,6 +575,7 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/>
+ <task xsi:type="configure" id="hdfs_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
@@ -630,6 +651,7 @@
<component name="RESOURCEMANAGER">
<pre-upgrade>
+ <task xsi:type="configure" id="yarn_log4j_parameterize" />
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig">
<summary>Calculating Yarn Properties for Spark Shuffle</summary>
</task>
@@ -659,6 +681,7 @@
<pre-upgrade>
<!-- These HBASE configs changed in HDP 2.3.4.0, but Ambari can't distinguish HDP 2.3.2.0 vs HDP 2.3.4.0, so easier to always do them. -->
<task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/>
+ <task xsi:type="configure" id="hbase_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
@@ -814,7 +837,7 @@
<component name="OOZIE_SERVER">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
-
+ <task xsi:type="configure" id="oozie_log4j_parameterize" />
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation">
<summary>Adjusting Oozie properties</summary>
</task>
@@ -863,6 +886,10 @@
<service name="FALCON">
<component name="FALCON_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="falcon_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade/>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -876,6 +903,10 @@
<service name="KAFKA">
<component name="KAFKA_BROKER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="kafka_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade></pre-downgrade>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -884,6 +915,10 @@
<service name="KNOX">
<component name="KNOX_GATEWAY">
+ <pre-upgrade>
+ <task xsi:type="configure" id="knox_gateway_log4j_parameterize" />
+ <task xsi:type="configure" id="knox_ldap_log4j_parameterize" />
+ </pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
<upgrade>
@@ -963,5 +998,17 @@
</upgrade>
</component>
</service>
+
+ <service name="ATLAS">
+ <component name="ATLAS_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="atlas_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade />
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
</processing>
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/21e8e4bd/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index d525520..eafb129 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -558,6 +558,10 @@
<processing>
<service name="ZOOKEEPER">
<component name="ZOOKEEPER_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="zookeeper_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade/>
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -573,6 +577,7 @@
<service name="RANGER">
<component name="RANGER_ADMIN">
<pre-upgrade>
+ <task xsi:type="configure" id="admin_log4j_parameterize" />
<task xsi:type="execute" hosts="all">
<summary>Stop Ranger Admin</summary>
<script>scripts/ranger_admin.py</script>
@@ -621,6 +626,20 @@
</component>
<component name="RANGER_USERSYNC">
+ <pre-upgrade>
+ <task xsi:type="configure" id="usersync_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade/>
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+
+ <component name="RANGER_TAGSYNC">
+ <pre-upgrade>
+ <task xsi:type="configure" id="tagsync_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade />
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -631,7 +650,7 @@
<component name="RANGER_KMS_SERVER">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db" />
-
+ <task xsi:type="configure" id="kms_log4j_parameterize" />
<task xsi:type="execute" hosts="any" sequential="true">
<summary>Upgrading Ranger KMS database schema</summary>
<script>scripts/kms_server.py</script>
@@ -658,6 +677,7 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" />
+ <task xsi:type="configure" id="hdfs_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
@@ -733,6 +753,7 @@
<component name="RESOURCEMANAGER">
<pre-upgrade>
+ <task xsi:type="configure" id="yarn_log4j_parameterize" />
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig">
<summary>Calculating Yarn Properties for Spark Shuffle</summary>
</task>
@@ -770,6 +791,7 @@
<!-- These HBASE configs changed in HDP 2.3.4.0, but Ambari can't distinguish HDP 2.3.2.0 vs HDP 2.3.4.0, so easier to always do them. -->
<task xsi:type="configure" id="hdp_2_4_0_0_hbase_remove_local_indexing"/>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hbase_audit_db" />
+ <task xsi:type="configure" id="hbase_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
@@ -944,7 +966,7 @@
<component name="OOZIE_SERVER">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
-
+ <task xsi:type="configure" id="oozie_log4j_parameterize" />
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation">
<summary>Adjusting Oozie properties</summary>
</task>
@@ -998,6 +1020,7 @@
<component name="FALCON_SERVER">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"/>
+ <task xsi:type="configure" id="falcon_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade/>
<upgrade>
@@ -1016,6 +1039,7 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kafka_audit_db" />
<task xsi:type="configure" id="hdp_2_5_0_0_add_protocol_compatibility" />
+ <task xsi:type="configure" id="kafka_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade/>
@@ -1030,6 +1054,8 @@
<component name="KNOX_GATEWAY">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_knox_audit_db" />
+ <task xsi:type="configure" id="knox_gateway_log4j_parameterize" />
+ <task xsi:type="configure" id="knox_ldap_log4j_parameterize" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
@@ -1193,5 +1219,18 @@
</upgrade>
</component>
</service>
+
+ <service name="ATLAS">
+ <component name="ATLAS_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="atlas_log4j_parameterize" />
+ </pre-upgrade>
+ <pre-downgrade />
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
</processing>
</upgrade>
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/21e8e4bd/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index 136fafe..db848c1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -139,6 +139,37 @@
<transfer operation="delete" delete-key="bind_anonymous" />
</definition>
+ <definition xsi:type="configure" id="admin_log4j_parameterize" summary="Parameterizing Ranger Log4J Properties">
+ <type>admin-log4j</type>
+ <set key="ranger_xa_log_maxfilesize" value="256"/>
+ <set key="ranger_xa_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.xa_log_appender.MaxFileSize={{ranger_xa_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.xa_log_appender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.xa_log_appender.MaxBackupIndex={{ranger_xa_log_maxbackupindex}}"/>
+ </definition>
+
+ </changes>
+ </component>
+
+ <component name="RANGER_USERSYNC">
+ <changes>
+ <definition xsi:type="configure" id="usersync_log4j_parameterize" summary="Parameterizing Ranger Usersync Log4J Properties">
+ <type>usersync-log4j</type>
+ <set key="ranger_usersync_log_maxfilesize" value="256"/>
+ <set key="ranger_usersync_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxFileSize = {{ranger_usersync_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxBackupIndex = {{ranger_usersync_log_maxbackupindex}}"/>
+ </definition>
+ </changes>
+ </component>
+ <component name="RANGER_TAGSYNC">
+ <changes>
+ <definition xsi:type="configure" id="tagsync_log4j_parameterize" summary="Parameterizing Ranger Tagsync Log4J Properties">
+ <type>tagsync-log4j</type>
+ <set key="ranger_tagsync_log_maxfilesize" value="256"/>
+ <set key="ranger_tagsync_log_number_of_backup_files" value="20"/>
+ <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxFileSize = {{ranger_tagsync_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.logFile=org.apache.log4j.DailyRollingFileAppender
log4j.appender.logFile.MaxBackupIndex = {{ranger_tagsync_log_number_of_backup_files}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -156,6 +187,17 @@
<transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" />
<transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" />
</definition>
+ <definition xsi:type="configure" id="kms_log4j_parameterize" summary="Parameterizing Ranger KMS Log4J Properties">
+ <type>kms-log4j</type>
+ <set key="ranger_kms_log_maxfilesize" value="256"/>
+ <set key="ranger_kms_log_maxbackupindex" value="20"/>
+ <set key="ranger_kms_audit_log_maxfilesize" value="256"/>
+ <set key="ranger_kms_audit_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms.MaxFileSize = {{ranger_kms_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms.MaxBackupIndex = {{ranger_kms_log_maxbackupindex}}"/>
+ <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms-audit.MaxFileSize = {{ranger_kms_audit_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kms-audit.MaxBackupIndex = {{ranger_kms_audit_log_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -179,6 +221,18 @@
<transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" />
<transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" />
</definition>
+ <!-- HDFS Rolling properties for log4j need to be parameterized. -->
+ <definition xsi:type="configure" id="hdfs_log4j_parameterize" summary="Parameterizing Hdfs Log4J Properties">
+ <type>hdfs-log4j</type>
+ <set key="hadoop_log_max_backup_size" value="256"/>
+ <set key="hadoop_log_number_of_backup_files" value="10"/>
+ <set key="hadoop_security_log_max_backup_size" value="256"/>
+ <set key="hadoop_security_log_number_of_backup_files" value="20"/>
+ <regex-replace key="content" find="log4j.appender.RFA.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RFA.MaxFileSize={{hadoop_log_max_backup_size}}MB"/>
+ <regex-replace key="content" find="log4j.appender.RFA.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RFA.MaxBackupIndex={{hadoop_log_number_of_backup_files}}"/>
+ <regex-replace key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/>
+ <regex-replace key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -196,6 +250,14 @@
<transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" />
<transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" />
</definition>
+ <!-- Yarn Rolling properties for log4j need to be parameterized. -->
+ <definition xsi:type="configure" id="yarn_log4j_parameterize" summary="Parameterizing Yarn Log4J Properties">
+ <type>yarn-log4j</type>
+ <set key="yarn_rm_summary_log_max_backup_size" value="256"/>
+ <set key="yarn_rm_summary_log_number_of_backup_files" value="20"/>
+ <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/>
+ <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/>
+ </definition>
</changes>
</component>
@@ -228,6 +290,17 @@
<set key="inter.broker.protocol.version" value="0.9.0.0" />
<set key="log.message.format.version" value="0.9.0.0" />
</definition>
+ <definition xsi:type="configure" id="kafka_log4j_parameterize" summary="Parameterizing Kafka Log4J Properties">
+ <type>kafka-log4j</type>
+ <set key="kafka_log_maxfilesize" value="256"/>
+ <set key="kafka_log_maxbackupindex" value="20"/>
+ <set key="controller_log_maxfilesize" value="256"/>
+ <set key="controller_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kafkaAppender.MaxFileSize = {{kafka_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kafkaAppender.MaxBackupIndex = {{kafka_log_maxbackupindex}}"/>
+ <replace key="content" find="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.controllerAppender.MaxFileSize = {{controller_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.controllerAppender.MaxBackupIndex = {{controller_log_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -325,6 +398,18 @@
<transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" />
<transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" />
</definition>
+ <!-- HBase Rolling properties for log4j need to be parameterized. -->
+ <definition xsi:type="configure" id="hbase_log4j_parameterize" summary="Parameterizing HBase Log4J Properties">
+ <type>hbase-log4j</type>
+ <set key="hbase_log_maxfilesize" value="256"/>
+ <set key="hbase_log_maxbackupindex" value="20"/>
+ <set key="hbase_security_log_maxfilesize" value="256"/>
+ <set key="hbase_security_log_maxbackupindex" value="20"/>
+ <regex-replace key="content" find="hbase.log.maxfilesize=([0-9]+)MB" replace-with="hbase.log.maxfilesize={{hbase_log_maxfilesize}}MB"/>
+ <regex-replace key="content" find="hbase.log.maxbackupindex=([0-9]+)" replace-with="hbase.log.maxbackupindex={{hbase_log_maxbackupindex}}"/>
+ <regex-replace key="content" find="hbase.security.log.maxfilesize=([0-9]+)MB" replace-with="hbase.security.log.maxfilesize={{hbase_security_log_maxfilesize}}MB"/>
+ <regex-replace key="content" find="hbase.security.log.maxbackupindex=([0-9]+)" replace-with="hbase.security.log.maxbackupindex={{hbase_security_log_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -342,6 +427,20 @@
<transfer operation="delete" delete-key="xasecure.audit.credential.provider.file" />
<transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir" />
</definition>
+ <definition xsi:type="configure" id="knox_gateway_log4j_parameterize" summary="Parameterizing Knox Gateway Log4J Properties">
+ <type>gateway-log4j</type>
+ <set key="knox_gateway_log_maxfilesize" value="256"/>
+ <set key="knox_gateway_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxFileSize = {{knox_gateway_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxBackupIndex = {{knox_gateway_log_maxbackupindex}}"/>
+ </definition>
+ <definition xsi:type="configure" id="knox_ldap_log4j_parameterize" summary="Parameterizing Knox Ldap Log4J Properties">
+ <type>ldap-log4j</type>
+ <set key="knox_ldap_log_maxfilesize" value="256"/>
+ <set key="knox_ldap_log_maxbackupindex" value="20"/>
+ <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxFileSize = {{knox_ldap_log_maxfilesize}}MB"/>
+ <replace key="content" find="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender
log4j.appender.drfa.MaxBackupIndex = {{knox_ldap_log_maxbackupindex}}"/>
+ </definition>
</changes>
</component>
</service>
@@ -353,6 +452,17 @@
<type>falcon-startup.properties</type>
<set key="*.application.services" value="org.apache.falcon.security.AuthenticationInitializationService, org.apache.falcon.workflow.WorkflowJobEndNotificationService, org.apache.falcon.service.ProcessSubscriberService, org.apache.falcon.extensions.ExtensionService, org.apache.falcon.service.LifecyclePolicyMap, org.apache.falcon.entity.store.ConfigurationStore, org.apache.falcon.rerun.service.RetryService, org.apache.falcon.rerun.service.LateRunService, org.apache.falcon.service.LogCleanupService, org.apache.falcon.metadata.MetadataMappingService{{atlas_application_class_addition}}"/>
</definition>
+ <definition xsi:type="configure" id="falcon_log4j_parameterize" summary="Parameterizing Falcon Log4J Properties">
+ <type>falcon-log4j</type>
+ <set key="falcon_log_maxfilesize" value="256"/>
+ <set key="falcon_log_maxbackupindex" value="20"/>
+ <set key="falcon_security_log_maxfilesize" value="256"/>
+ <set key="falcon_security_log_maxbackupindex" value="20"/>
+ <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxFileSize" value="{{falcon_log_maxfilesize}}MB" />"/>
+ <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxBackupIndex" value="{{falcon_log_maxbackupindex}}" />"/>
+ <replace key="content" find="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxFileSize" value="{{falcon_security_log_maxfilesize}}MB"/>"/>
+ <replace key="content" find="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="SECURITY" class="org.apache.log4j.DailyRollingFileAppender">
<param name="MaxBackupIndex" value="{{falcon_security_log_maxbackupindex}}"/>"/>
+ </definition>
</changes>
</component>
</service>
@@ -376,5 +486,48 @@
</component>
</service>
+ <service name="OOZIE">
+ <component name="OOZIE_SERVER">
+ <changes>
+ <!-- Oozie Rolling properties for log4j need to be parameterized. -->
+ <definition xsi:type="configure" id="oozie_log4j_parameterize" summary="Parameterizing Oozie Log4J Properties">
+ <type>oozie-log4j</type>
+ <set key="oozie_log_maxhistory" value="720"/>
+ <regex-replace key="content" find="^log4j.appender.oozie.RollingPolicy.MaxHistory=([0-9]+)" replace-with="log4j.appender.oozie.RollingPolicy.MaxHistory={{oozie_log_maxhistory}}"/>
+ </definition>
+ </changes>
+ </component>
+ </service>
+
+ <service name="ZOOKEEPER">
+ <component name="ZOOKEEPER_SERVER">
+ <changes>
+ <!-- Zookeeper Rolling properties for log4j need to be parameterized. -->
+ <definition xsi:type="configure" id="zookeeper_log4j_parameterize" summary="Parameterizing ZooKeeper Log4J Properties">
+ <type>zookeeper-log4j</type>
+ <set key="zookeeper_log_max_backup_size" value="10"/>
+ <set key="zookeeper_log_number_of_backup_files" value="10"/>
+ <regex-replace key="content" find="^log4j.appender.ROLLINGFILE.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.ROLLINGFILE.MaxFileSize={{zookeeper_log_max_backup_size}}MB"/>
+ <regex-replace key="content" find="^#log4j.appender.ROLLINGFILE.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.ROLLINGFILE.MaxBackupIndex={{zookeeper_log_number_of_backup_files}}"/>
+ </definition>
+ </changes>
+ </component>
+ </service>
+
+ <service name="ATLAS">
+ <component name="ATLAS_SERVER">
+ <changes>
+ <definition xsi:type="configure" id="atlas_log4j_parameterize" summary="Parameterizing Atlas Log4J Properties">
+ <type>atlas-log4j</type>
+ <set key="atlas_log_max_backup_size" value="256"/>
+ <set key="atlas_log_number_of_backup_files" value="20"/>
+ <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">\n<param name="MaxFileSize" value="{{atlas_log_max_backup_size}}MB" />"/>
+ <replace key="content" find="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">" replace-with="<appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">\n<param name="MaxFileSize" value="{{atlas_log_number_of_backup_files}}" />"/>
+ </definition>
+ </changes>
+ </component>
+ </service>
+
+
</services>
</upgrade-config-changes>