You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2017/05/30 18:29:26 UTC
[04/50] [abbrv] ambari git commit: AMBARI-20999. More updates to llap
configs for hdp stack (Siddharth via smohanty)
AMBARI-20999. More updates to llap configs for hdp stack (Siddharth via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f7e9af87
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f7e9af87
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f7e9af87
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: f7e9af87c75d8641a96b8b097b7b44f0a9f13538
Parents: 023c819
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu May 11 17:03:27 2017 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu May 11 17:10:35 2017 -0700
----------------------------------------------------------------------
.../server/upgrade/UpgradeCatalog250.java | 72 --------------------
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 15 ++++
.../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 12 ++++
.../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 2 +
.../configuration/hive-interactive-site.xml | 11 ++-
.../HIVE/configuration/tez-interactive-site.xml | 13 ++++
.../server/upgrade/UpgradeCatalog250Test.java | 23 +------
7 files changed, 52 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7e9af87/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index 058acb2..1740e25 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -200,7 +200,6 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
updateHadoopEnvConfigs();
updateKafkaConfigs();
updateHIVEInteractiveConfigs();
- updateHiveLlapConfigs();
unInstallAllZeppelinViews();
updateZeppelinConfigs();
updateAtlasConfigs();
@@ -495,56 +494,6 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
}
}
- protected void updateHiveLlapConfigs() throws AmbariException {
- AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
- Clusters clusters = ambariManagementController.getClusters();
-
- if (clusters != null) {
- Map<String, Cluster> clusterMap = clusters.getClusters();
-
- if (clusterMap != null && !clusterMap.isEmpty()) {
- for (final Cluster cluster : clusterMap.values()) {
- Set<String> installedServices = cluster.getServices().keySet();
-
- if (installedServices.contains("HIVE")) {
- Config hiveSite = cluster.getDesiredConfigByType(HIVE_INTERACTIVE_SITE);
- if (hiveSite != null) {
- Map<String, String> hiveSiteProperties = hiveSite.getProperties();
- String schedulerDelay = hiveSiteProperties.get("hive.llap.task.scheduler.locality.delay");
- if (schedulerDelay != null) {
- // Property exists. Change to new default if set to -1.
- if (schedulerDelay.length() != 0) {
- try {
- int schedulerDelayInt = Integer.parseInt(schedulerDelay);
- if (schedulerDelayInt == -1) {
- // Old default. Set to new default.
- updateConfigurationProperties(HIVE_INTERACTIVE_SITE, Collections
- .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
- false);
- }
- } catch (NumberFormatException e) {
- // Invalid existing value. Set to new default.
- updateConfigurationProperties(HIVE_INTERACTIVE_SITE, Collections
- .singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
- false);
- }
- }
- }
- updateConfigurationProperties(HIVE_INTERACTIVE_SITE,
- Collections.singletonMap("hive.mapjoin.hybridgrace.hashtable", "true"), true,
- false);
- updateConfigurationProperties("tez-interactive-site",
- Collections.singletonMap("tez.session.am.dag.submit.timeout.secs", "1209600"), true,
- false);
- // Explicitly skipping hive.llap.allow.permanent.fns during upgrades, since it's related to security,
- // and we don't know if the value is set by the user or as a result of the previous default.
- }
- }
- }
- }
- }
- }
-
protected void updateAMSConfigs() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Clusters clusters = ambariManagementController.getClusters();
@@ -1050,7 +999,6 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
*
* @throws AmbariException
*/
- private static final String HIVE_INTERACTIVE_SITE = "hive-interactive-site";
private static final String HIVE_INTERACTIVE_ENV = "hive-interactive-env";
private static final String HIVE_ENV = "hive-env";
protected void updateHIVEInteractiveConfigs() throws AmbariException {
@@ -1061,26 +1009,6 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
if (clusterMap != null && !clusterMap.isEmpty()) {
for (final Cluster cluster : clusterMap.values()) {
- Config hiveInteractiveSite = cluster.getDesiredConfigByType(HIVE_INTERACTIVE_SITE);
- if (hiveInteractiveSite != null) {
- Map<String, String> newProperties = new HashMap<>();
- newProperties.put("hive.auto.convert.join.noconditionaltask.size", "1000000000");
-
- String llapRpcPortString = hiveInteractiveSite.getProperties().get("hive.llap.daemon.rpc.port");
- if (StringUtils.isNotBlank(llapRpcPortString)) {
- try {
- int llapRpcPort = Integer.parseInt(llapRpcPortString);
- if (llapRpcPort == 15001) {
- newProperties.put("hive.llap.daemon.rpc.port", "0");
- LOG.info("Updating HSI hive.llap.daemon.rpc.port to: 0");
- }
- } catch (NumberFormatException e) {
- LOG.warn("Unable to parse llap.rpc.port as integer: " + llapRpcPortString);
- }
- }
- updateConfigurationProperties(HIVE_INTERACTIVE_SITE, newProperties, true, true);
- }
-
Config hiveInteractiveEnv = cluster.getDesiredConfigByType(HIVE_INTERACTIVE_ENV);
Config hiveEnv = cluster.getDesiredConfigByType(HIVE_ENV);
if (hiveInteractiveEnv != null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7e9af87/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 559d8dc..045ed5a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -480,6 +480,21 @@
<set key="hive.vectorized.groupby.maxentries" value="1000000"/>
</definition>
+ <definition xsi:type="configure" id="llap_update_settings" summary="Update additional LLAP settings">
+ <type>hive-interactive-site</type>
+ <set key="hive.llap.task.scheduler.locality.delay" value="-1"/>
+ <set key="hive.mapjoin.hybridgrace.hashtable" value="false"/>
+ <set key="hive.merge.nway.joins" value="false"/>
+ <set key="hive.llap.daemon.rpc.port" value="0"/>
+ </definition>
+
+ <definition xsi:type="configure" id="llap_update_tez_settings" summary="Update additional LLAP-Tez settings">
+ <type>tez-interactive-site</type>
+ <set key="tez.runtime.shuffle.keep-alive.enabled" value="true"/>
+ <set key="tez.am.am-rm.heartbeat.interval-ms.max" value="30000"/>
+ <set key="tez.session.am.dag.submit.timeout.secs" value="1209600"/>
+ </definition>
+
<definition xsi:type="configure" id="hdp_2_6_0_0_copy_hive_tez_container_size_to_hiveInteractive">
<type>hive-interactive-site</type>
<transfer operation="copy" from-type="hive-site" from-key="hive.tez.container.size" to-key="hive.tez.container.size" default-value="682" if-type="hive-interactive-site" if-key="hive.tez.container.size" if-key-state="absent"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7e9af87/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index 25a8568..8c659ee 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -562,6 +562,18 @@
</task>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Update additional LLAP settings">
+ <task xsi:type="configure" id="llap_update_settings">
+ <summary>Update additional LLAP settings</summary>
+ </task>
+ </execute-stage>
+
+ <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Update additional LLAP-Tez settings">
+ <task xsi:type="configure" id="llap_update_tez_settings">
+ <summary>Update additional LLAP-Tez settings</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Update JVM stack size for LLAP">
<task xsi:type="configure" id="llap_append_stack_size_java_opts">
<summary>Updating JVM stack size for LLAP</summary>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7e9af87/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index 0470b7a..3054ca3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -845,6 +845,8 @@
<task xsi:type="configure" id="llap_update_hashaggregation"/>
<task xsi:type="configure" id="llap_append_stack_size_java_opts"/>
<task xsi:type="configure" id="llap_update_shuffle_parallel_copies"/>
+ <task xsi:type="configure" id="llap_update_settings"/>
+ <task xsi:type="configure" id="llap_update_tez_settings"/>
</pre-upgrade>
<pre-downgrade />
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7e9af87/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
index 0610833..cb4f947 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
@@ -30,7 +30,7 @@ limitations under the License.
-->
<property>
<name>hive.llap.task.scheduler.locality.delay</name>
- <value>8000</value>
+ <value>-1</value>
<description>
Amount of time to wait (in ms) before allocating a request which contains location information,
to a location other than the ones requested. Set to -1 for an infinite delay, 0
@@ -41,7 +41,7 @@ limitations under the License.
<property>
<name>hive.mapjoin.hybridgrace.hashtable</name>
- <value>true</value>
+ <value>false</value>
<description>Whether to use hybrid grace hash join as the join method for mapjoin.
Applies to dynamically partitioned joins when running in LLAP, but not to regular
broadcast(map) joins. hive.llap.enable.grace.join.in.llap is used for this.
@@ -103,4 +103,11 @@ limitations under the License.
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>hive.merge.nway.joins</name>
+ <value>false</value>
+ <description>Merge adjacent joins into a single n-way join</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7e9af87/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
index ab1202a..09416e3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/tez-interactive-site.xml
@@ -102,4 +102,17 @@
</description>
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>tez.runtime.shuffle.keep-alive.enabled</name>
+ <value>true</value>
+ <description>Connection keep-alive for shuffle</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>tez.am.am-rm.heartbeat.interval-ms.max</name>
+ <value>10000</value>
+ <description>The heartbeat interval between the tez AM and YARN RM</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f7e9af87/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
index b5dc32b..3cb2c47 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
@@ -386,7 +386,6 @@ public class UpgradeCatalog250Test {
Method updateAmsConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAMSConfigs");
Method updateHadoopEnvConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHadoopEnvConfigs");
Method updateKafkaConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateKafkaConfigs");
- Method updateHiveLlapConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHiveLlapConfigs");
Method updateTablesForZeppelinViewRemoval = UpgradeCatalog250.class.getDeclaredMethod("unInstallAllZeppelinViews");
Method updateHIVEInteractiveConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateHIVEInteractiveConfigs");
Method addManageServiceAutoStartPermissions = UpgradeCatalog250.class.getDeclaredMethod("addManageServiceAutoStartPermissions");
@@ -409,7 +408,6 @@ public class UpgradeCatalog250Test {
.addMockedMethod(updateAmsConfigs)
.addMockedMethod(updateHadoopEnvConfigs)
.addMockedMethod(updateKafkaConfigs)
- .addMockedMethod(updateHiveLlapConfigs)
.addMockedMethod(addNewConfigurationsFromXml)
.addMockedMethod(addManageServiceAutoStartPermissions)
.addMockedMethod(addManageAlertNotificationsPermissions)
@@ -444,9 +442,6 @@ public class UpgradeCatalog250Test {
upgradeCatalog250.updateHIVEInteractiveConfigs();
expectLastCall().once();
- upgradeCatalog250.updateHiveLlapConfigs();
- expectLastCall().once();
-
upgradeCatalog250.unInstallAllZeppelinViews();
expectLastCall().once();
@@ -1704,19 +1699,6 @@ public class UpgradeCatalog250Test {
"hive.metastore.heapsize", "512",
"hive_ambari_database", "MySQL");
- Map<String, String> oldHiveIntSite = ImmutableMap.of(
- "hive.llap.daemon.rpc.port", "15001");
-
- Map<String, String> expectedHiveIntSite = ImmutableMap.of(
- "hive.llap.daemon.rpc.port", "0",
- "hive.auto.convert.join.noconditionaltask.size", "1000000000");
-
- Config mockHsiSite = easyMockSupport.createNiceMock(Config.class);
- expect(cluster.getDesiredConfigByType("hive-interactive-site")).andReturn(mockHsiSite).atLeastOnce();
- expect(mockHsiSite.getProperties()).andReturn(oldHiveIntSite).anyTimes();
- Capture<Map<String, String>> hsiSiteCapture = EasyMock.newCapture();
- expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(hsiSiteCapture), anyString(),
- EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
Config mockHiveEnv = easyMockSupport.createNiceMock(Config.class);
expect(cluster.getDesiredConfigByType("hive-env")).andReturn(mockHiveEnv).atLeastOnce();
@@ -1731,13 +1713,10 @@ public class UpgradeCatalog250Test {
replay(clusters, cluster);
replay(controller, injector2);
- replay(mockHsiEnv, mockHiveEnv, mockHsiSite);
+ replay(mockHsiEnv, mockHiveEnv);
new UpgradeCatalog250(injector2).updateHIVEInteractiveConfigs();
easyMockSupport.verifyAll();
- Map<String, String> updatedHsiSite = hsiSiteCapture.getValue();
- assertTrue(Maps.difference(expectedHiveIntSite, updatedHsiSite).areEqual());
-
Map<String, String> updatedHsiEnv = hsiEnvCapture.getValue();
assertTrue(Maps.difference(expectedHsiEnv, updatedHsiEnv).areEqual());
}