You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2016/12/01 17:06:47 UTC
[02/32] ambari git commit: AMBARI-18792. Update some configuration
properties for hive interactive for the HDP 2.6 stack (Siddharth Seth via
smohanty)
AMBARI-18792. Update some configuration properties for hive interactive for the HDP 2.6 stack (Siddharth Seth via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ca7e071f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ca7e071f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ca7e071f
Branch: refs/heads/branch-feature-AMBARI-18456
Commit: ca7e071f044d70186c7bf9d18bf24c29f3767730
Parents: 2cc4c9e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Nov 29 11:39:53 2016 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Nov 29 15:28:46 2016 -0800
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog250.java | 52 +++++++++++++
.../configuration/hive-interactive-site.xml | 58 +++++++++++++++
.../HIVE/configuration/tez-interactive-site.xml | 78 ++++++++++++++++++++
3 files changed, 188 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca7e071f/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index e81568c..3b2cdd3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -23,6 +23,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.CommandExecutionType;
@@ -134,6 +135,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
addNewConfigurationsFromXml();
updateAMSConfigs();
updateKafkaConfigs();
+ updateHiveLlapConfigs();
}
protected void updateHostVersionTable() throws SQLException {
@@ -151,6 +153,56 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
dbAccessor.addUniqueConstraint(GROUPS_TABLE, "UNQ_groups_0", "group_name", "group_type");
}
+ protected void updateHiveLlapConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+ Set<String> installedServices = cluster.getServices().keySet();
+
+ if (installedServices.contains("HIVE")) {
+ Config hiveSite = cluster.getDesiredConfigByType("hive-interactive-site");
+ if (hiveSite != null) {
+ Map<String, String> hiveSiteProperties = hiveSite.getProperties();
+ String schedulerDelay = hiveSiteProperties.get("hive.llap.task.scheduler.locality.delay");
+ if (schedulerDelay != null) {
+ // Property exists. Change to new default if set to -1.
+ if (schedulerDelay.length() != 0) {
+ try {
+ int schedulerDelayInt = Integer.parseInt(schedulerDelay);
+ if (schedulerDelayInt == -1) {
+ // Old default. Set to new default.
+ updateConfigurationProperties("hive-interactive-site", Collections
+ .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+ false);
+ }
+ } catch (NumberFormatException e) {
+ // Invalid existing value. Set to new default.
+ updateConfigurationProperties("hive-interactive-site", Collections
+ .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+ false);
+ }
+ }
+ }
+ updateConfigurationProperties("hive-interactive-site",
+ Collections.singletonMap("hive.mapjoin.hybridgrace.hashtable", "true"), true,
+ false);
+ updateConfigurationProperties("tez-interactive-site",
+ Collections.singletonMap("tez.session.am.dag.submit.timeout.secs", "1209600"), true,
+ false);
+ // Explicitly skipping hive.llap.allow.permanent.fns during upgrades, since it's related to security,
+ // and we don't know if the value is set by the user or as a result of the previous default.
+ }
+ }
+ }
+ }
+ }
+ }
+
protected void updateAMSConfigs() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Clusters clusters = ambariManagementController.getClusters();
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca7e071f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
new file mode 100644
index 0000000..f33ade7
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<configuration supports_final="true">
+ <!--
+ Note: This file includes only those configs which are supposed to have different value from the parent hive/hive-site.
+ It inherits the other required configs from hive/hive-site.
+ The inheritance logic in order to get hive2/hive-site goes like this :
+
+ 1. We read the hive/hive-site which includes the stack defaults and Stack Advisor recommended values.
+ 2. We take the read hive/hive-site (step 1), and on top of it apply the hive-interactive-site to get
+ hive2/hive-site.
+
+ Therefore, any config defined here will override the config value read from hive2/hive-site (step 1).
+ -->
+ <property>
+ <name>hive.llap.task.scheduler.locality.delay</name>
+ <value>8000</value>
+ <description>
+ Amount of time to wait (in ms) before allocating a request which contains location information,
+ to a location other than the ones requested. Set to -1 for an infinite delay, 0
+ for no delay.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>hive.mapjoin.hybridgrace.hashtable</name>
+ <value>true</value>
+ <description>Whether to use hybrid grace hash join as the join method for mapjoin. Tez only.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>hive.llap.enable.grace.join.in.llap</name>
+ <value>false</value>
+ <description>Override if grace join should be allowed to run in llap for regular map joins.
+ Dynamic partitioned joins will honor the hive.mapjoin.hybridgrace.hashtable property in LLAP
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca7e071f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
new file mode 100644
index 0000000..f7de121
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
@@ -0,0 +1,78 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration supports_final="true">
+ <!--
+ Note: This file includes only those configs which are supposed to have different value from the parent tez/tez-site.
+ It inherits the other required configs from tez/tez-site.
+ The inheritance logic in order to get tez_hive2/tez-site goes like this :
+
+ 1. We read the tez/tez-site which includes the stack defaults and Stack Advisor recommended values.
+ 2. We take the read tez/tez-site (step 1), and on top of it apply the tez-interactive-site to get tez_hive2/tez-site.
+
+ Therefore, any config defined here will override the config value read from tez/tez-site (step 1).
+ -->
+ <property>
+ <name>tez.task.heartbeat.timeout.check-ms</name>
+ <value>11000</value>
+ <description>The time interval, in milliseconds, at which the AM will check
+ for timed out tasks
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>tez.task.timeout-ms</name>
+ <value>20000</value>
+ <description>Amount of time the Tez AM waits before marking a task which
+ has not sent in a heartbeat, as timed out
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>tez.am.client.heartbeat.timeout.secs</name>
+ <value>60</value>
+ <description>The time interval, after which an AM will kill itself, if it
+ does not receive a heartbeat from the client.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>tez.am.client.heartbeat.poll.interval.millis</name>
+ <value>6000</value>
+ <description>The interval at which the AM checks for a client heartbeat
+ timeout
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>tez.am.node-blacklisting.enabled</name>
+ <value>false</value>
+ <description>Whether to enable blacklisting in Tez AMs. Disable for LLAP
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>tez.session.am.dag.submit.timeout.secs</name>
+ <value>1209600</value>
+ <description>The amoung of time an AM will wait, before killing itself,
+ if not DAG is submitted.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>