You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2016/11/21 20:37:23 UTC
[2/2] ambari git commit: AMBARI-18954. Use 'Number of LLAP Nodes'
selected as the driver for LLAP config calculations.
AMBARI-18954. Use 'Number of LLAP Nodes' selected as the driver for LLAP config calculations.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4942deaf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4942deaf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4942deaf
Branch: refs/heads/branch-feature-AMBARI-18901
Commit: 4942deaf5bd79f35beeaeb6f47844a93881502ea
Parents: 4fcc493
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Mon Nov 21 12:37:04 2016 -0800
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Mon Nov 21 12:37:04 2016 -0800
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog250.java | 71 +-
.../HIVE/configuration/hive-interactive-env.xml | 42 +-
.../configuration/hive-interactive-site.xml | 68 +-
.../HIVE/configuration/tez-interactive-site.xml | 71 +-
.../HDP/2.5/services/HIVE/themes/theme.json | 27 +-
.../YARN/configuration/capacity-scheduler.xml | 2 +-
.../stacks/HDP/2.5/services/stack_advisor.py | 665 +++++++++++--------
.../server/upgrade/UpgradeCatalog250Test.java | 287 +++++++-
8 files changed, 854 insertions(+), 379 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index bc3c120..6de48c1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -17,13 +17,8 @@
*/
package org.apache.ambari.server.upgrade;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
+import com.google.inject.Inject;
+import com.google.inject.Injector;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.actionmanager.CommandExecutionType;
import org.apache.ambari.server.controller.AmbariManagementController;
@@ -37,8 +32,12 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
/**
* Upgrade catalog for version 2.5.0.
@@ -134,6 +133,8 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
addNewConfigurationsFromXml();
updateAMSConfigs();
updateKafkaConfigs();
+ updateHIVEInteractiveConfigs();
+ updateTEZInteractiveConfigs();
}
protected void updateHostVersionTable() throws SQLException {
@@ -269,5 +270,57 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
dbAccessor.addColumn(SERVICE_DESIRED_STATE_TABLE,
new DBColumnInfo(CREDENTIAL_STORE_ENABLED_COL, Short.class, null, 0, false));
}
+
+ /**
+ * Updates Hive Interactive's config in hive-interactive-site.
+ *
+ * @throws AmbariException
+ */
+ protected void updateHIVEInteractiveConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+ Config hiveInteractiveSite = cluster.getDesiredConfigByType("hive-interactive-site");
+ if (hiveInteractiveSite != null) {
+ updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.tez.container.size",
+ "SET_ON_FIRST_INVOCATION"), true, true);
+
+ updateConfigurationProperties("hive-interactive-site", Collections.singletonMap("hive.auto.convert.join.noconditionaltask.size",
+ "1000000000"), true, true);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Updates Tez for Hive2 Interactive's config in tez-interactive-site.
+ *
+ * @throws AmbariException
+ */
+ protected void updateTEZInteractiveConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+ Config tezInteractiveSite = cluster.getDesiredConfigByType("tez-interactive-site");
+ if (tezInteractiveSite != null) {
+
+ updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.io.sort.mb", "512"), true, true);
+
+ updateConfigurationProperties("tez-interactive-site", Collections.singletonMap("tez.runtime.unordered.output.buffer.size-mb",
+ "100"), true, true);
+ }
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
index 1fd72eb..5a186b1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-env.xml
@@ -53,40 +53,14 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>llap_queue_capacity</name>
- <value>0</value>
- <description>Percentage of the cluster dedicated to interactive query.</description>
- <display-name>% of Cluster Capacity</display-name>
- <value-attributes>
- <type>int</type>
- <minimum>20</minimum>
- <maximum>100</maximum>
- <increment-step>1</increment-step>
- </value-attributes>
- <depends-on>
- <property>
- <type>hive-interactive-env</type>
- <name>enable_hive_interactive</name>
- </property>
- <property>
- <type>hive-interactive-site</type>
- <name>hive.llap.daemon.queue.name</name>
- </property>
- <property>
- <type>capacity-scheduler</type>
- <name>yarn.scheduler.capacity.root.queues</name>
- </property>
- </depends-on>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
<name>num_llap_nodes</name>
<value>1</value>
<description>The number of Hive LLAP daemons to run.</description>
- <display-name>Number of LLAP Daemons</display-name>
+ <display-name>Number of LLAP Nodes</display-name>
<value-attributes>
<type>int</type>
<minimum>1</minimum>
+ <increment-step>1</increment-step>
</value-attributes>
<depends-on>
<property>
@@ -94,10 +68,6 @@
<name>enable_hive_interactive</name>
</property>
<property>
- <type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
- </property>
- <property>
<type>hive-interactive-site</type>
<name>hive.llap.daemon.queue.name</name>
</property>
@@ -105,10 +75,6 @@
<type>capacity-scheduler</type>
<name>yarn.scheduler.capacity.root.queues</name>
</property>
- <property>
- <type>hive-interactive-site</type>
- <name>hive.server2.tez.sessions.per.default.queue</name>
- </property>
</depends-on>
<on-ambari-upgrade add="true"/>
</property>
@@ -140,7 +106,7 @@
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
@@ -177,7 +143,7 @@
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 0207e49..2fb1553 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -67,7 +67,7 @@ limitations under the License.
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
@@ -411,7 +411,7 @@ limitations under the License.
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
@@ -447,7 +447,7 @@ limitations under the License.
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
@@ -478,7 +478,7 @@ limitations under the License.
<name>hive.llap.daemon.yarn.container.mb</name>
<display-name>Memory per daemon</display-name>
<description>Total memory used by individual LLAP daemons. This includes memory for the cache as well as for the query execution.</description>
- <value>341</value>
+ <value>0</value>
<value-attributes>
<type>int</type>
<unit>MB</unit>
@@ -491,7 +491,7 @@ limitations under the License.
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
@@ -646,4 +646,62 @@ limitations under the License.
</value-attributes>
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>hive.tez.container.size</name>
+ <value>SET_ON_FIRST_INVOCATION</value>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>hive.auto.convert.join.noconditionaltask.size</name>
+ <value>1000000000</value>
+ <description>If hive.auto.convert.join.noconditionaltask is off, this parameter does not take affect. However, if it
+ is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller than this size, the join is directly
+ converted to a mapjoin(there is no conditional task). The default is 10MB.
+ </description>
+ <depends-on>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
index 9e588e9..b331736 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml
@@ -83,11 +83,8 @@
</property>
<property>
<name>tez.am.resource.memory.mb</name>
- <value>1536</value>
+ <value>0</value>
<description>The amount of memory to be used by the AppMaster</description>
- <value-attributes>
- <type>int</type>
- </value-attributes>
<depends-on>
<property>
<type>hive-interactive-env</type>
@@ -95,7 +92,71 @@
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>tez.runtime.io.sort.mb</name>
+ <value>512</value>
+ <description>The size of the sort buffer when output needs to be sorted</description>
+ <depends-on>
+ <property>
+ <type>tez-site</type>
+ <name>tez.task.resource.memory.mb</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.llap.daemon.queue.name</name>
+ </property>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ <property>
+ <type>hive-interactive-site</type>
+ <name>hive.server2.tez.sessions.per.default.queue</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>tez.runtime.unordered.output.buffer.size-mb</name>
+ <value>100</value>
+ <description>The size of the buffer when output does not require to be sorted</description>
+ <depends-on>
+ <property>
+ <type>tez-site</type>
+ <name>tez.task.resource.memory.mb</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>enable_hive_interactive</name>
+ </property>
+ <property>
+ <type>hive-interactive-env</type>
+ <name>num_llap_nodes</name>
</property>
<property>
<type>hive-interactive-site</type>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
index 452537d..1d50b6a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/themes/theme.json
@@ -63,11 +63,7 @@
]
},
{
- "config": "hive-interactive-env/llap_queue_capacity",
- "subsection-name": "interactive-query-row1-col1"
- },
- {
- "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue",
+ "config": "hive-interactive-env/num_llap_nodes",
"subsection-name": "interactive-query-row1-col1",
"depends-on": [
{
@@ -89,12 +85,8 @@
]
},
{
- "config": "hive-interactive-env/copy_num_llap_nodes",
+ "config": "hive-interactive-site/hive.server2.tez.sessions.per.default.queue",
"subsection-name": "interactive-query-row1-col1",
- "property_value_attributes": {
- "ui_only_property": true,
- "copy": "hive-interactive-env/num_llap_nodes"
- },
"depends-on": [
{
"configs":[
@@ -226,23 +218,12 @@
}
},
{
- "config": "hive-interactive-env/copy_num_llap_nodes",
- "widget": {
- "type": "label",
- "units": [
- {
- "unit-name": "int"
- }
- ]
- }
- },
- {
- "config": "hive-interactive-env/llap_queue_capacity",
+ "config": "hive-interactive-env/num_llap_nodes",
"widget": {
"type": "slider",
"units": [
{
- "unit-name": "percent"
+ "unit-name": "int"
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/4942deaf/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
index 9ff8484..bc0ecc4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/capacity-scheduler.xml
@@ -26,7 +26,7 @@
</property>
<property>
<type>hive-interactive-env</type>
- <name>llap_queue_capacity</name>
+ <name>num_llap_nodes</name>
</property>
</depends-on>
<on-ambari-upgrade add="false"/>