You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2015/04/15 03:48:02 UTC
ambari git commit: AMBARI-10480. BE: Extend stack-advisor to
recommend property_value_attributes for Hive/Tez (dsen via srimanth)
Repository: ambari
Updated Branches:
refs/heads/trunk ab52fb399 -> 8938de8ae
AMBARI-10480. BE: Extend stack-advisor to recommend property_value_attributes for Hive/Tez (dsen via srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8938de8a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8938de8a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8938de8a
Branch: refs/heads/trunk
Commit: 8938de8ae757900b79738de90ab46320db1b54aa
Parents: ab52fb3
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Tue Apr 14 17:19:20 2015 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Tue Apr 14 18:47:13 2015 -0700
----------------------------------------------------------------------
.../recommendations/RecommendationResponse.java | 7 +-
.../stacks/HDP/2.0.6/services/stack_advisor.py | 2 +-
.../services/HIVE/configuration/hive-site.xml | 6 +
.../stacks/HDP/2.1/services/stack_advisor.py | 2 +-
.../services/HIVE/configuration/hive-env.xml | 26 +-
.../services/HIVE/configuration/hive-site.xml | 24 +-
.../stacks/HDP/2.2/services/stack_advisor.py | 183 +++++++++++--
.../services/YARN/configuration/yarn-site.xml | 2 +-
.../stacks/2.2/common/test_stack_advisor.py | 255 +++++++++++++++++--
.../mixins/common/configs/enhanced_configs.js | 4 +-
10 files changed, 452 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8938de8a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
index b81320d..fdd7f96 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/recommendations/RecommendationResponse.java
@@ -22,6 +22,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorResponse;
+import org.apache.ambari.server.state.ValueAttributesInfo;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -117,7 +118,7 @@ public class RecommendationResponse extends StackAdvisorResponse {
@JsonProperty("property_attributes")
@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL)
- private Map<String, Map<String, String>> propertyAttributes;
+ private Map<String, ValueAttributesInfo> propertyAttributes;
public BlueprintConfigurations() {
System.out.println(this);
@@ -131,11 +132,11 @@ public class RecommendationResponse extends StackAdvisorResponse {
this.properties = properties;
}
- public Map<String, Map<String, String>> getPropertyAttributes() {
+ public Map<String, ValueAttributesInfo> getPropertyAttributes() {
return propertyAttributes;
}
- public void setPropertyAttributes(Map<String, Map<String, String>> propertyAttributes) {
+ public void setPropertyAttributes(Map<String, ValueAttributesInfo> propertyAttributes) {
this.propertyAttributes = propertyAttributes;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8938de8a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 38d6eda..860f6a2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -117,7 +117,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
config[configType]["property_attributes"] = {}
if key not in config[configType]["property_attributes"]:
config[configType]["property_attributes"][key] = {}
- config[configType]["property_attributes"][key][attribute] = str(attributeValue)
+ config[configType]["property_attributes"][key][attribute] = attributeValue if isinstance(attributeValue, list) else str(attributeValue)
return appendPropertyAttribute
def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
http://git-wip-us.apache.org/repos/asf/ambari/blob/8938de8a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
index d22176d..ad0b8bb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
@@ -369,6 +369,12 @@ limitations under the License.
<unit>MB</unit>
<increment-step>682</increment-step>
</value-attributes>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ </property>
+ </depends-on>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8938de8a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 52816e9..2f2ca40 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -40,7 +40,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
def recommendHiveConfigurations(self, configurations, clusterData, services, hosts):
containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize)
- putHiveProperty = self.putProperty(configurations, "hive-site")
+ putHiveProperty = self.putProperty(configurations, "hive-site", services)
putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(round(containerSize / 3)) * 1048576)
putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(round((0.8 * containerSize) + 0.5)))
+ "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps")
http://git-wip-us.apache.org/repos/asf/ambari/blob/8938de8a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
index 66c0b66..f849c87 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
@@ -85,16 +85,36 @@ export METASTORE_PORT={{hive_metastore_port}}
<property>
<name>hive_txn_acid</name>
<display-name>ACID Transactions</display-name>
- <value>true</value>
+ <value>Off</value>
<value-attributes>
<type>value-list</type>
<entries>
<entry>
- <value>true</value>
+ <value>On</value>
<label>On</label>
</entry>
<entry>
- <value>false</value>
+ <value>Off</value>
+ <label>Off</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>cost_based_optimizer</name>
+ <display-name>Cost Based Optimizer</display-name>
+ <value>Off</value>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>On</value>
+ <label>On</label>
+ </entry>
+ <entry>
+ <value>Off</value>
<label>Off</label>
</entry>
</entries>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8938de8a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
index 74cae89..95fd405 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
@@ -205,7 +205,7 @@ limitations under the License.
<depends-on>
<property>
<type>hive-env</type>
- <name>hive_timeline_logging_enabled </name>
+ <name>hive_timeline_logging_enabled</name>
</property>
</depends-on>
</property>
@@ -221,7 +221,7 @@ limitations under the License.
<depends-on>
<property>
<type>hive-env</type>
- <name>hive_timeline_logging_enabled </name>
+ <name>hive_timeline_logging_enabled</name>
</property>
</depends-on>
</property>
@@ -237,7 +237,7 @@ limitations under the License.
<depends-on>
<property>
<type>hive-env</type>
- <name>hive_timeline_logging_enabled </name>
+ <name>hive_timeline_logging_enabled</name>
</property>
</depends-on>
</property>
@@ -412,6 +412,12 @@ limitations under the License.
<name>hive.cbo.enable</name>
<value>true</value>
<description>Flag to control enabling Cost Based Optimizations using Calcite framework.</description>
+ <depends-on>
+ <property>
+ <type>hive-env</type>
+ <name>cost_based_optimizer</name>
+ </property>
+ </depends-on>
</property>
<property>
@@ -753,6 +759,12 @@ limitations under the License.
</entries>
<selection-cardinality>1</selection-cardinality>
</value-attributes>
+ <depends-on>
+ <property>
+ <type>hive-env</type>
+ <name>cost_based_optimizer</name>
+ </property>
+ </depends-on>
</property>
<property>
<name>hive.stats.fetch.column.stats</name>
@@ -778,6 +790,12 @@ limitations under the License.
</entries>
<selection-cardinality>1</selection-cardinality>
</value-attributes>
+ <depends-on>
+ <property>
+ <type>hive-env</type>
+ <name>cost_based_optimizer</name>
+ </property>
+ </depends-on>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8938de8a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index bcb9404..e5f8b5c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -19,7 +19,6 @@ limitations under the License.
import math
-
class HDP22StackAdvisor(HDP21StackAdvisor):
def getServiceConfigurationRecommenderDict(self):
@@ -45,12 +44,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
nodeManagerHost = self.getHostWithComponent("YARN", "NODEMANAGER", services, hosts)
if (nodeManagerHost is not None):
putYarnProperty('yarn.nodemanager.resource.cpu-vcores', nodeManagerHost["Hosts"]["cpu_count"] * 2)
- putYarnPropertyAttribute('yarn.nodemanager.resource.memory-mb', 'max', int(nodeManagerHost["Hosts"]["total_mem"] / 1024)) # total_mem in kb
- putYarnPropertyAttribute('yarn.nodemanager.resource.cpu-vcores', 'max', nodeManagerHost["Hosts"]["cpu_count"] * 4)
- putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-vcores', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"])
- putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"])
- putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
- putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'max', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
+ putYarnPropertyAttribute('yarn.nodemanager.resource.memory-mb', 'maximum', int(nodeManagerHost["Hosts"]["total_mem"] / 1024)) # total_mem in kb
+ putYarnPropertyAttribute('yarn.nodemanager.resource.cpu-vcores', 'maximum', nodeManagerHost["Hosts"]["cpu_count"] * 4)
+ putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-vcores', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"])
+ putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-vcores', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.cpu-vcores"])
+ putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
+ putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
putHdfsSiteProperty = self.putProperty(configurations, "hdfs-site", services)
@@ -95,7 +94,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
else:
namenode_heapsize = int(namenodeHosts[0]["Hosts"]["total_mem"] / 1024) # total_mem in kb
- putHdfsEnvPropertyAttribute('namenode_heapsize', 'max', namenode_heapsize)
+ putHdfsEnvPropertyAttribute('namenode_heapsize', 'maximum', namenode_heapsize)
datanodeHosts = self.getHostsWithComponent("HDFS", "DATANODE", services, hosts)
if (datanodeHosts is not None and len(datanodeHosts)>0):
@@ -103,25 +102,156 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
for datanode in datanodeHosts:
ram_kb = datanode['Hosts']['total_mem']
min_datanode_ram_kb = min(min_datanode_ram_kb, ram_kb)
- putHdfsEnvPropertyAttribute('dtnode_heapsize', 'max', int(min_datanode_ram_kb/1024))
+ putHdfsEnvPropertyAttribute('dtnode_heapsize', 'maximum', int(min_datanode_ram_kb/1024))
putHdfsSitePropertyAttribute = self.putPropertyAttribute(configurations, "hdfs-site")
- putHdfsSitePropertyAttribute('dfs.datanode.failed.volumes.tolerated', 'max', dataDirsCount)
+ putHdfsSitePropertyAttribute('dfs.datanode.failed.volumes.tolerated', 'maximum', dataDirsCount)
def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts):
super(HDP22StackAdvisor, self).recommendHiveConfigurations(configurations, clusterData, services, hosts)
+
+ putHiveServerProperty = self.putProperty(configurations, "hiveserver2-site", services)
+ putHiveEnvProperty = self.putProperty(configurations, "hive-env", services)
+ putHiveSiteProperty = self.putProperty(configurations, "hive-site", services)
+
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
if 'ranger-hive-plugin-properties' in services['configurations'] and ('ranger-hive-plugin-enabled' in services['configurations']['ranger-hive-plugin-properties']['properties']):
rangerPluginEnabled = services['configurations']['ranger-hive-plugin-properties']['properties']['ranger-hive-plugin-enabled']
if ("RANGER" in servicesList) :
if (rangerPluginEnabled.lower() == "Yes".lower()):
- putHiveProperty = self.putProperty(configurations, "hiveserver2-site")
- putHiveProperty("hive.security.authorization.manager", 'com.xasecure.authorization.hive.authorizer.XaSecureHiveAuthorizerFactory')
- putHiveProperty("hive.security.authenticator.manager", 'org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator')
+ putHiveServerProperty("hive.security.authorization.manager", 'com.xasecure.authorization.hive.authorizer.XaSecureHiveAuthorizerFactory')
+ putHiveServerProperty("hive.security.authenticator.manager", 'org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator')
elif (rangerPluginEnabled.lower() == "No".lower()):
- putHiveProperty = self.putProperty(configurations, "hiveserver2-site")
- putHiveProperty("hive.security.authorization.manager", 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory')
- putHiveProperty("hive.security.authenticator.manager", 'org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator')
+ putHiveServerProperty("hive.security.authorization.manager", 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory')
+ putHiveServerProperty("hive.security.authenticator.manager", 'org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator')
+
+ # Storage
+ putHiveEnvProperty("hive_exec_orc_storage_strategy", "SPEED")
+ putHiveSiteProperty("hive.exec.orc.encoding.strategy", configurations["hive-env"]["properties"]["hive_exec_orc_storage_strategy"])
+ putHiveSiteProperty("hive.exec.orc.compression.strategy", configurations["hive-env"]["properties"]["hive_exec_orc_storage_strategy"])
+
+ putHiveSiteProperty("hive.exec.orc.default.stripe.size", "67108864")
+ putHiveSiteProperty("hive.exec.orc.default.compress", "ZLIB")
+ putHiveSiteProperty("hive.optimize.index.filter", "true")
+ putHiveSiteProperty("hive.optimize.sort.dynamic.partition", "false")
+
+ # Vectorization
+ putHiveSiteProperty("hive.vectorized.execution.enabled", "true")
+ putHiveSiteProperty("hive.vectorized.execution.reduce.enabled", "false")
+
+ # Memory
+ putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", "2147483648")
+ putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864")
+
+ # Transactions
+ putHiveEnvProperty("hive_txn_acid", "false")
+ if str(configurations["hive-env"]["properties"]["hive_txn_acid"]).lower() == "on":
+ putHiveSiteProperty("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager")
+ putHiveSiteProperty("hive.support.concurrency", "true")
+ putHiveSiteProperty("hive.compactor.initiator.on", "true")
+ putHiveSiteProperty("hive.compactor.worker.threads", "1")
+ putHiveSiteProperty("hive.enforce.bucketing", "true")
+ putHiveSiteProperty("hive.exec.dynamic.partition.mode", "nostrict")
+ else:
+ putHiveSiteProperty("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager")
+ putHiveSiteProperty("hive.support.concurrency", "false")
+ putHiveSiteProperty("hive.compactor.initiator.on", "false")
+ putHiveSiteProperty("hive.compactor.worker.threads", "0")
+ putHiveSiteProperty("hive.enforce.bucketing", "false")
+ putHiveSiteProperty("hive.exec.dynamic.partition.mode", "strict")
+
+ # ATS
+ putHiveEnvProperty("hive_timeline_logging_enabled", "true")
+
+ hooks_properties = ["hive.exec.pre.hooks", "hive.exec.post.hooks", "hive.exec.failure.hooks"]
+ include_ats_hook = str(configurations["hive-env"]["properties"]["hive_timeline_logging_enabled"]).lower() == "true"
+
+ ats_hook_class = "org.apache.hadoop.hive.ql.hooks.ATSHook"
+ for hooks_property in hooks_properties:
+ if hooks_property in configurations["hive-site"]["properties"]:
+ hooks_value = configurations["hive-site"]["properties"][hooks_property]
+ else:
+ hooks_value = " "
+ if include_ats_hook and ats_hook_class not in hooks_value:
+ if hooks_value == " ":
+ hooks_value = ats_hook_class
+ else:
+ hooks_value = hooks_value + "," + ats_hook_class
+ if not include_ats_hook and ats_hook_class in hooks_value:
+ hooks_classes = []
+ for hook_class in hooks_value.split(","):
+ if hook_class != ats_hook_class and hook_class != " ":
+ hooks_classes.append(hook_class)
+ if hooks_classes:
+ hooks_value = ",".join(hooks_classes)
+ else:
+ hooks_value = " "
+
+ putHiveSiteProperty(hooks_property, hooks_value)
+
+ # Tez Engine
+ if "TEZ" in servicesList:
+ putHiveSiteProperty("hive.execution.engine", "tez")
+ else:
+ putHiveSiteProperty("hive.execution.engine", "mr")
+
+ container_size = "512"
+
+ if "yarn-site" in configurations and \
+ "yarn.scheduler.minimum-allocation-mb" in configurations["yarn-site"]["properties"]:
+ container_size = configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]
+ putHiveSiteProperty("hive.tez.container.size", container_size)
+ putHiveSiteProperty("hive.prewarm.enabled", "false")
+ putHiveSiteProperty("hive.prewarm.numcontainers", "3")
+ putHiveSiteProperty("hive.tez.auto.reducer.parallelism", "true")
+ putHiveSiteProperty("hive.tez.dynamic.partition.pruning", "true")
+
+ # CBO
+ putHiveEnvProperty("cost_based_optimizer", "On")
+ if str(configurations["hive-env"]["properties"]["cost_based_optimizer"]).lower() == "on":
+ putHiveSiteProperty("hive.cbo.enable", "true")
+ putHiveSiteProperty("hive.stats.fetch.partition.stats", "true")
+ putHiveSiteProperty("hive.stats.fetch.column.stats", "true")
+ else:
+ putHiveSiteProperty("hive.cbo.enable", "false")
+ putHiveSiteProperty("hive.stats.fetch.partition.stats", "false")
+ putHiveSiteProperty("hive.stats.fetch.column.stats", "false")
+ putHiveSiteProperty("hive.compute.query.using.stats ", "true")
+
+ # Interactive Query
+ putHiveServerProperty("hive.server2.tez.initialize.default.sessions", "false")
+ putHiveServerProperty("hive.server2.tez.sessions.per.default.queue", "1")
+ putHiveServerProperty("hive.server2.enable.doAs", "true")
+ putHiveServerProperty("tez.session.am.dag.submit.timeout.secs", "600")
+
+ yarn_queues = "default"
+ if "capacity-scheduler" in configurations and \
+ "yarn.scheduler.capacity.root.queues" in configurations["capacity-scheduler"]["properties"]:
+ yarn_queues = str(configurations["capacity-scheduler"]["properties"]["yarn.scheduler.capacity.root.queues"])
+ putHiveServerProperty("hive.server2.tez.default.queues", yarn_queues)
+
+ # Interactive Queues property attributes
+ putHiveServerPropertyAttribute = self.putPropertyAttribute(configurations, "hiveserver2-site")
+ entries = []
+ for queue in yarn_queues.split(","):
+ entries.append({"label": str(queue) + " queue", "value": queue})
+ putHiveServerPropertyAttribute("hive.server2.tez.default.queues", "entries", entries)
+
+ # Security
+ putHiveEnvProperty("hive_security_authorization", "None")
+ if str(configurations["hive-env"]["properties"]["hive_security_authorization"]).lower() == "none":
+ putHiveSiteProperty("hive.security.authorization.enabled", "false")
+ else:
+ putHiveSiteProperty("hive.security.authorization.enabled", "true")
+
+ if str(configurations["hive-env"]["properties"]["hive_security_authorization"]).lower() == "sqlstdauth":
+ auth_manager_value = str(configurations["hive-env"]["properties"]["hive.security.metastore.authorization.manager"])
+ sqlstdauth_class = "org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly"
+ if sqlstdauth_class not in auth_manager_value:
+ putHiveSiteProperty("hive.security.metastore.authorization.manager", auth_manager_value + "," + sqlstdauth_class)
+
+ putHiveServerProperty("hive.server2.enable.doAs", "true")
+ putHiveSiteProperty("hive.server2.use.SSL", "false")
def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
super(HDP22StackAdvisor, self).recommendHbaseEnvConfigurations(configurations, clusterData, services, hosts)
@@ -134,7 +264,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
host_ram = host["Hosts"]["total_mem"]
min_ram = min(min_ram, host_ram)
- putHbaseEnvPropertyAttributes('hbase_regionserver_heapsize', 'max', max(1024, int(min_ram*0.8/1024)))
+ putHbaseEnvPropertyAttributes('hbase_regionserver_heapsize', 'maximum', max(1024, int(min_ram*0.8/1024)))
putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
putHbaseSiteProperty("hbase.regionserver.global.memstore.upperLimit", '0.4')
@@ -232,12 +362,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putMapredPropertyAttribute = self.putPropertyAttribute(configurations, "mapred-site")
yarnMinAllocationSize = int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"])
yarnMaxAllocationSize = min(30 * int(configurations["yarn-site"]["properties"]["yarn.scheduler.minimum-allocation-mb"]), int(configurations["yarn-site"]["properties"]["yarn.scheduler.maximum-allocation-mb"]))
- putMapredPropertyAttribute("mapreduce.map.memory.mb", "max", yarnMaxAllocationSize)
- putMapredPropertyAttribute("mapreduce.map.memory.mb", "min", yarnMinAllocationSize)
- putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "max", yarnMaxAllocationSize)
- putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "min", yarnMinAllocationSize)
- putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "max", yarnMaxAllocationSize)
- putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "min", yarnMinAllocationSize)
+ putMapredPropertyAttribute("mapreduce.map.memory.mb", "maximum", yarnMaxAllocationSize)
+ putMapredPropertyAttribute("mapreduce.map.memory.mb", "minimum", yarnMinAllocationSize)
+ putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "maximum", yarnMaxAllocationSize)
+ putMapredPropertyAttribute("mapreduce.reduce.memory.mb", "minimum", yarnMinAllocationSize)
+ putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "maximum", yarnMaxAllocationSize)
+ putMapredPropertyAttribute("yarn.app.mapreduce.am.resource.mb", "minimum", yarnMinAllocationSize)
def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
@@ -449,6 +579,13 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
"item": self.getWarnItem(
"If Ranger Hive Plugin is enabled." \
" {0} needs to contain {1}".format(prop_name, ','.join(missing_vals)))})
+ stripe_size_values = [8388608, 16777216, 33554432, 67108864, 134217728, 268435456]
+ stripe_size_property = "hive.exec.orc.default.stripe.size"
+ if int(properties[stripe_size_property]) not in stripe_size_values:
+ validationItems.append({"config-name": stripe_size_property,
+ "item": self.getWarnItem("Correct values are ")
+ }
+ )
return self.toConfigurationValidationProblems(validationItems, "hive-site")
def validateHBASEConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
http://git-wip-us.apache.org/repos/asf/ambari/blob/8938de8a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-site.xml
index e64d7fe..29968f6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-site.xml
@@ -33,7 +33,7 @@
<description>
Enabled node labels to restrict YARN applications so that they run only on cluster nodes that have a specified node label.
</description>
- <dispaly-name>Enable Node Labels</dispaly-name>
+ <display-name>Enable Node Labels</display-name>
<value-attributes>
<type>boolean</type>
</value-attributes>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8938de8a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 5e9b48c..8a721d1 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -678,12 +678,12 @@ class TestHDP22StackAdvisor(TestCase):
"yarn.nodemanager.resource.cpu-vcores": "2"
},
"property_attributes": {
- 'yarn.nodemanager.resource.memory-mb': {'max': '1877'},
- 'yarn.nodemanager.resource.cpu-vcores': {'max': '4'},
- 'yarn.scheduler.minimum-allocation-vcores': {'max': '2'},
- 'yarn.scheduler.maximum-allocation-vcores': {'max': '2'},
- 'yarn.scheduler.minimum-allocation-mb': {'max': '1280'},
- 'yarn.scheduler.maximum-allocation-mb': {'max': '1280'}
+ 'yarn.nodemanager.resource.memory-mb': {'maximum': '1877'},
+ 'yarn.nodemanager.resource.cpu-vcores': {'maximum': '4'},
+ 'yarn.scheduler.minimum-allocation-vcores': {'maximum': '2'},
+ 'yarn.scheduler.maximum-allocation-vcores': {'maximum': '2'},
+ 'yarn.scheduler.minimum-allocation-mb': {'maximum': '1280'},
+ 'yarn.scheduler.maximum-allocation-mb': {'maximum': '1280'}
}
}
}
@@ -827,6 +827,217 @@ class TestHDP22StackAdvisor(TestCase):
self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
+ def test_recommendHiveConfigurationAttributes(self):
+ configurations = {
+ "yarn-site": {
+ "properties": {
+ "yarn.scheduler.minimum-allocation-mb": "256",
+ },
+ },
+ "capacity-scheduler": {
+ "properties": {
+ "yarn.scheduler.capacity.root.queues": "queue1,queue2"
+ }
+ }
+ }
+ clusterData = {
+ "cpu": 4,
+ "mapMemory": 3000,
+ "amMemory": 2000,
+ "reduceMemory": 2056,
+ "containers": 3,
+ "ramPerContainer": 256
+ }
+ expected = {
+ 'capacity-scheduler': {
+ 'properties': {
+ 'yarn.scheduler.capacity.root.queues': 'queue1,queue2'
+ }
+ },
+ 'yarn-site': {
+ 'properties': {
+ 'yarn.scheduler.minimum-allocation-mb': '256'
+ }
+ },
+ 'hive-env': {
+ 'properties': {
+ 'cost_based_optimizer': 'On',
+ 'hive_exec_orc_storage_strategy': 'SPEED',
+ 'hive_security_authorization': 'None',
+ 'hive_timeline_logging_enabled': 'true',
+ 'hive_txn_acid': 'false'
+ }
+ },
+ 'hive-site': {
+ 'properties': {
+ 'hive.auto.convert.join.noconditionaltask.size': '2147483648',
+ 'hive.cbo.enable': 'true',
+ 'hive.compactor.initiator.on': 'false',
+ 'hive.compactor.worker.threads': '0',
+ 'hive.compute.query.using.stats ': 'true',
+ 'hive.enforce.bucketing': 'false',
+ 'hive.exec.dynamic.partition.mode': 'strict',
+ 'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
+ 'hive.exec.orc.compression.strategy': 'SPEED',
+ 'hive.exec.orc.default.compress': 'ZLIB',
+ 'hive.exec.orc.default.stripe.size': '67108864',
+ 'hive.exec.orc.encoding.strategy': 'SPEED',
+ 'hive.exec.post.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
+ 'hive.exec.pre.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
+ 'hive.exec.reducers.bytes.per.reducer': '67108864',
+ 'hive.execution.engine': 'mr',
+ 'hive.optimize.index.filter': 'true',
+ 'hive.optimize.sort.dynamic.partition': 'false',
+ 'hive.prewarm.enabled': 'false',
+ 'hive.prewarm.numcontainers': '3',
+ 'hive.security.authorization.enabled': 'false',
+ 'hive.server2.use.SSL': 'false',
+ 'hive.stats.fetch.column.stats': 'true',
+ 'hive.stats.fetch.partition.stats': 'true',
+ 'hive.support.concurrency': 'false',
+ 'hive.tez.auto.reducer.parallelism': 'true',
+ 'hive.tez.container.size': '256',
+ 'hive.tez.dynamic.partition.pruning': 'true',
+ 'hive.tez.java.opts': '-server -Xmx615m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps',
+ 'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager',
+ 'hive.vectorized.execution.enabled': 'true',
+ 'hive.vectorized.execution.reduce.enabled': 'false'
+ }
+ },
+ 'hiveserver2-site': {
+ 'properties': {
+ 'hive.server2.enable.doAs': 'true',
+ 'hive.server2.tez.default.queues': "queue1,queue2",
+ 'hive.server2.tez.initialize.default.sessions': 'false',
+ 'hive.server2.tez.sessions.per.default.queue': '1',
+ 'tez.session.am.dag.submit.timeout.secs': '600'
+ },
+ 'property_attributes': {
+ 'hive.server2.tez.default.queues': {
+ 'entries': [{'value': 'queue1', 'label': 'queue1 queue'}, {'value': 'queue2', 'label': 'queue2 queue'}]
+ }
+ }
+ },
+ }
+ services = {
+ "services": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN",
+ "StackServices": {
+ "service_name": "YARN",
+ "service_version": "2.6.0.2.2",
+ "stack_name": "HDP",
+ "stack_version": "2.2"
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "advertise_version": "false",
+ "cardinality": "1",
+ "component_category": "MASTER",
+ "component_name": "APP_TIMELINE_SERVER",
+ "display_name": "App Timeline Server",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": []
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "SLAVE",
+ "component_name": "NODEMANAGER",
+ "display_name": "NodeManager",
+ "is_client": "false",
+ "is_master": "false",
+ "hostnames": [
+ "c6403.ambari.apache.org"
+ ]
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1-2",
+ "component_category": "MASTER",
+ "component_name": "RESOURCEMANAGER",
+ "display_name": "ResourceManager",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": []
+ },
+ "dependencies": []
+ },
+ {
+ "StackServiceComponents": {
+ "advertise_version": "true",
+ "cardinality": "1+",
+ "component_category": "CLIENT",
+ "component_name": "YARN_CLIENT",
+ "display_name": "YARN Client",
+ "is_client": "true",
+ "is_master": "false",
+ "hostnames": []
+ },
+ "dependencies": []
+ }
+ ]
+ },
+ ],
+ "configurations": configurations,
+ "changed-configurations": [ ]
+
+ }
+ hosts = {
+ "items" : [
+ {
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6401.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6401.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6402.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6402.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6402.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ },
+ {
+ "href" : "/api/v1/hosts/c6403.ambari.apache.org",
+ "Hosts" : {
+ "cpu_count" : 1,
+ "host_name" : "c6403.ambari.apache.org",
+ "os_arch" : "x86_64",
+ "os_type" : "centos6",
+ "ph_cpu_count" : 1,
+ "public_host_name" : "c6403.ambari.apache.org",
+ "rack_info" : "/default-rack",
+ "total_mem" : 1922680
+ }
+ }
+ ]
+ }
+
+ self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations, expected)
+
def test_recommendMapredConfigurationAttributes(self):
configurations = {
"mapred-site": {
@@ -870,12 +1081,12 @@ class TestHDP22StackAdvisor(TestCase):
"mapreduce.task.io.sort.mb": "70"
},
"property_attributes": {
- 'yarn.app.mapreduce.am.resource.mb': {'max': '1280',
- 'min': '100'},
- 'mapreduce.map.memory.mb': {'max': '1280',
- 'min': '100'},
- 'mapreduce.reduce.memory.mb': {'max': '1280',
- 'min': '100'}
+ 'yarn.app.mapreduce.am.resource.mb': {'maximum': '1280',
+ 'minimum': '100'},
+ 'mapreduce.map.memory.mb': {'maximum': '1280',
+ 'minimum': '100'},
+ 'mapreduce.reduce.memory.mb': {'maximum': '1280',
+ 'minimum': '100'}
}
},
"yarn-site": {
@@ -886,12 +1097,12 @@ class TestHDP22StackAdvisor(TestCase):
"yarn.nodemanager.resource.cpu-vcores": "2"
},
"property_attributes": {
- 'yarn.nodemanager.resource.memory-mb': {'max': '1877'},
- 'yarn.nodemanager.resource.cpu-vcores': {'max': '4'},
- 'yarn.scheduler.minimum-allocation-vcores': {'max': '2'},
- 'yarn.scheduler.maximum-allocation-vcores': {'max': '2'},
- 'yarn.scheduler.minimum-allocation-mb': {'max': '1280'},
- 'yarn.scheduler.maximum-allocation-mb': {'max': '1280'}
+ 'yarn.nodemanager.resource.memory-mb': {'maximum': '1877'},
+ 'yarn.nodemanager.resource.cpu-vcores': {'maximum': '4'},
+ 'yarn.scheduler.minimum-allocation-vcores': {'maximum': '2'},
+ 'yarn.scheduler.maximum-allocation-vcores': {'maximum': '2'},
+ 'yarn.scheduler.minimum-allocation-mb': {'maximum': '1280'},
+ 'yarn.scheduler.maximum-allocation-mb': {'maximum': '1280'}
}
}
}
@@ -1054,7 +1265,7 @@ class TestHDP22StackAdvisor(TestCase):
}
]
}
- self.maxDiff = None
+
self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, services, hosts)
self.assertEquals(configurations, expected)
@@ -1171,8 +1382,8 @@ class TestHDP22StackAdvisor(TestCase):
'namenode_opt_maxnewsize' : '256'
},
'property_attributes': {
- 'dtnode_heapsize': {'max': '2048'},
- 'namenode_heapsize': {'max': '1024'}
+ 'dtnode_heapsize': {'maximum': '2048'},
+ 'namenode_heapsize': {'maximum': '1024'}
}
},
'hdfs-site': {
@@ -1184,7 +1395,7 @@ class TestHDP22StackAdvisor(TestCase):
'dfs.datanode.data.dir': '/path/1,/path/2,/path/3,/path/4'
},
'property_attributes': {
- 'dfs.datanode.failed.volumes.tolerated': {'max': '4'}
+ 'dfs.datanode.failed.volumes.tolerated': {'maximum': '4'}
}
},
'ranger-hdfs-plugin-properties': {
http://git-wip-us.apache.org/repos/asf/ambari/blob/8938de8a/ambari-web/app/mixins/common/configs/enhanced_configs.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/common/configs/enhanced_configs.js b/ambari-web/app/mixins/common/configs/enhanced_configs.js
index ecc096d..6fb63a7 100644
--- a/ambari-web/app/mixins/common/configs/enhanced_configs.js
+++ b/ambari-web/app/mixins/common/configs/enhanced_configs.js
@@ -443,10 +443,10 @@ App.EnhancedConfigsMixin = Em.Mixin.create({
var stackProperty = App.StackConfigProperty.find(propertyName + '_' + key);
if (stackProperty && stackProperty.get('valueAttributes')) {
if (configs[key].property_attributes[propertyName].min) {
- stackProperty.set('valueAttributes.minimum', configs[key].property_attributes[propertyName].min);
+ stackProperty.set('valueAttributes.minimum', configs[key].property_attributes[propertyName].minimum);
}
if (configs[key].property_attributes[propertyName].max) {
- stackProperty.set('valueAttributes.maximum', configs[key].property_attributes[propertyName].max);
+ stackProperty.set('valueAttributes.maximum', configs[key].property_attributes[propertyName].maximum);
}
if (configs[key].property_attributes[propertyName].increment_step) {
stackProperty.set('valueAttributes.increment_step', configs[key].property_attributes[propertyName].increment_step);