You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by am...@apache.org on 2017/11/16 15:37:11 UTC
[48/50] [abbrv] ambari git commit: AMBARI-22260. Update Spark2 log4j
default settings to latest. (sai.sai.shao via sshridhar).
AMBARI-22260. Update Spark2 log4j default settings to latest. (sai.sai.shao via sshridhar).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/579bf1c6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/579bf1c6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/579bf1c6
Branch: refs/heads/branch-feature-AMBARI-22008
Commit: 579bf1c65c857898455d209da3778d94ec72cbf0
Parents: 69bb45f
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Wed Nov 15 16:21:31 2017 -0800
Committer: Attila Magyar <am...@hortonworks.com>
Committed: Thu Nov 16 16:35:32 2017 +0100
----------------------------------------------------------------------
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 23 ++++++++
.../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 23 +++++---
.../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 45 +++++++++-------
.../configuration/spark2-log4j-properties.xml | 56 ++++++++++++++++++++
4 files changed, 122 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/579bf1c6/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index b1e5f91..8c2c860 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -104,14 +104,37 @@
</service>
<service name="SPARK2">
+ <component name="SPARK2_JOBHISTORYSERVER">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_jobhistoryserver_log4j">
+ <type>spark2-log4j-properties</type>
+ <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/>
+ <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR" replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/>
+ </definition>
+ </changes>
+ </component>
<component name="SPARK2_CLIENT">
<changes>
+ <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_client_log4j">
+ <type>spark2-log4j-properties</type>
+ <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/>
+ <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR" replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/>
+ </definition>
<definition xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue">
<type>spark2-defaults</type>
<set key="spark.yarn.queue" value="default" if-type="spark-defaults" if-key="spark.yarn.queue" if-key-state="absent"/>
</definition>
</changes>
</component>
+ <component name="SPARK2_THRIFTSERVER">
+ <changes>
+ <definition xsi:type="configure" id="hdp_2_5_0_0_spark2_thriftserver_log4j">
+ <type>spark2-log4j-properties</type>
+ <replace key="content" find="log4j.logger.org.eclipse.jetty=WARN" replace-with="log4j.logger.org.spark_project.jetty=WARN"/>
+ <replace key="content" find="log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR" replace-with="log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR"/>
+ </definition>
+ </changes>
+ </component>
</service>
<service name="TEZ">
http://git-wip-us.apache.org/repos/asf/ambari/blob/579bf1c6/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index ace9542..9caa850 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -77,7 +77,7 @@
<service name="FLUME">
<component>FLUME_HANDLER</component>
</service>
-
+
<service name="ACCUMULO">
<component>ACCUMULO_TRACER</component>
<component>ACCUMULO_GC</component>
@@ -355,7 +355,7 @@
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixCapacitySchedulerOrderingPolicy">
<summary>Validate Root Queue Ordering Policy</summary>
</task>
- </execute-stage>
+ </execute-stage>
<!--TEZ-->
<execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
@@ -679,6 +679,15 @@
<summary>Add queue customization property</summary>
</task>
</execute-stage>
+ <execute-stage service="SPARK2" component="SPARK2_THRIFTSERVER" title="Apply config changes for Spark2 ThriftServer log4j">
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark2_thriftserver_log4j"/>
+ </execute-stage>
+ <execute-stage service="SPARK2" component="SPARK2_JOBHISTORYSERVER" title="Apply config changes for Spark2 Job HistoryServer log4j">
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark2_jobhistoryserver_log4j"/>
+ </execute-stage>
+ <execute-stage service="SPARK2" component="SPARK2_CLIENT" title="Apply config changes for Spark2 Client log4j">
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark2_client_log4j"/>
+ </execute-stage>
</group>
<!--
@@ -720,7 +729,7 @@
</task>
</execute-stage>
</group>
-
+
<!-- Now, restart all of the services. -->
<group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
<service-check>false</service-check>
@@ -1083,7 +1092,7 @@
<component>FLUME_HANDLER</component>
</service>
</group>
-
+
<group xsi:type="restart" name="ACCUMULO" title="Accumulo">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -1112,7 +1121,7 @@
<group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check">
<direction>UPGRADE</direction>
-
+
<execute-stage title="Check Component Versions">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" />
</execute-stage>
@@ -1192,7 +1201,7 @@
<function>setup_ranger_java_patches</function>
</task>
</pre-upgrade>
-
+
<pre-downgrade copy-upgrade="true" />
<upgrade>
@@ -1215,7 +1224,7 @@
<function>configure_atlas_user_for_tagsync</function>
</task>
</pre-upgrade>
-
+
<pre-downgrade copy-upgrade="true" />
<upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/579bf1c6/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index df11ae1..4fc9b42 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -183,7 +183,7 @@
<component>HBASE_MASTER</component>
</service>
</group>
-
+
<group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check">
<direction>UPGRADE</direction>
<skippable>true</skippable>
@@ -199,7 +199,7 @@
<service>LOGSEARCH</service>
</exclude>
</group>
-
+
<group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -207,7 +207,7 @@
<component>DATANODE</component>
<component>NFS_GATEWAY</component>
</service>
-
+
<service name="HBASE">
<component>HBASE_REGIONSERVER</component>
<component>PHOENIX_QUERY_SERVER</component>
@@ -223,7 +223,7 @@
<message>The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services.</message>
</batch>
</group>
-
+
<group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check">
<direction>UPGRADE</direction>
<skippable>true</skippable>
@@ -458,7 +458,7 @@
<group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}} Pre-Check">
<direction>UPGRADE</direction>
-
+
<execute-stage title="Check Component Versions">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction" />
</execute-stage>
@@ -467,7 +467,7 @@
<group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
<skippable>true</skippable>
<supports-auto-skip-failure>false</supports-auto-skip-failure>
-
+
<execute-stage title="Confirm Finalize">
<direction>UPGRADE</direction>
<task xsi:type="manual">
@@ -491,7 +491,7 @@
<function>finalize_rolling_upgrade</function>
</task>
</execute-stage>
-
+
<execute-stage title="Save Cluster State">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction">
</task>
@@ -577,7 +577,7 @@
<function>configure_atlas_user_for_tagsync</function>
</task>
</pre-upgrade>
-
+
<pre-downgrade copy-upgrade="true" />
<upgrade>
@@ -668,7 +668,7 @@
</upgrade>
</component>
</service>
-
+
<service name="SLIDER">
<component name="SLIDER">
<upgrade>
@@ -691,7 +691,7 @@
</task>
<task xsi:type="configure" id="hdp_2_6_0_0_mapreduce_job_queuename"/>
</pre-upgrade>
-
+
<pre-downgrade copy-upgrade="true" />
<upgrade>
@@ -722,11 +722,11 @@
<task xsi:type="configure" id="yarn_site_retained_log_count" />
<task xsi:type="configure" id="hdp_2_6_0_0_service_check_queue_name"/>
<task xsi:type="configure" id="hdp_2_6_0_0_ats_scan_interval_default"/>
-
+
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixCapacitySchedulerOrderingPolicy">
<summary>Validate Root Queue Ordering Policy</summary>
</task>
-
+
</pre-upgrade>
<pre-downgrade />
<upgrade>
@@ -785,9 +785,9 @@
</task>
<task xsi:type="configure" id="hdp_2_5_0_0_tez_queue_name"/>
</pre-upgrade>
-
+
<pre-downgrade copy-upgrade="true" />
-
+
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -865,9 +865,9 @@
<task xsi:type="configure" id="hdp_2_6_maint_llap_config"/>
<task xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable"/>
</pre-upgrade>
-
+
<pre-downgrade />
-
+
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -913,9 +913,9 @@
<task xsi:type="configure" id="hdp_2_5_0_0_rename_spark_livy_configs" />
<task xsi:type="configure" id="hdp_2_5_0_0_add_spark_conf_dir_livy_configs" />
</pre-upgrade>
-
+
<pre-downgrade />
-
+
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -933,11 +933,19 @@
<service name="SPARK2">
<component name="SPARK2_JOBHISTORYSERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark2_jobhistoryserver_log4j"/>
+ </pre-upgrade>
+ <pre-downgrade />
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
</component>
<component name="SPARK2_THRIFTSERVER">
+ <pre-upgrade>
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark2_thriftserver_log4j"/>
+ </pre-upgrade>
+ <pre-downgrade />
<upgrade>
<task xsi:type="restart-task" />
</upgrade>
@@ -945,6 +953,7 @@
<component name="SPARK2_CLIENT">
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_5_0_0_spark2_yarn_queue"/>
+ <task xsi:type="configure" id="hdp_2_5_0_0_spark2_client_log4j"/>
</pre-upgrade>
<pre-downgrade/>
<upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/579bf1c6/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml
new file mode 100644
index 0000000..d7479f4
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-log4j-properties.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+ <property>
+ <name>content</name>
+ <description>Spark2-log4j-Properties</description>
+ <value>
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+
+# Set the default spark-shell log level to WARN. When running the spark-shell, the
+# log level for this class is used to overwrite the root logger's log level, so that
+# the user can have different defaults for the shell and regular Spark apps.
+log4j.logger.org.apache.spark.repl.Main=WARN
+
+# Settings to quiet third party logs that are too verbose
+log4j.logger.org.spark_project.jetty=WARN
+log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
+log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
+log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
+log4j.logger.org.apache.parquet=ERROR
+log4j.logger.parquet=ERROR
+
+# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
+log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
+log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
+ </value>
+ <value-attributes>
+ <type>content</type>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>