You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2016/02/11 10:33:19 UTC
ambari git commit: AMBARI-14989. Inconsistent HIVE CBO setting
presentation. (Jaimin)
Repository: ambari
Updated Branches:
refs/heads/branch-2.2 9377291da -> e8f3ee9d9
AMBARI-14989. Inconsistent HIVE CBO setting presentation. (Jaimin)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e8f3ee9d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e8f3ee9d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e8f3ee9d
Branch: refs/heads/branch-2.2
Commit: e8f3ee9d95675b20fea41eea7081fd72f092ef75
Parents: 9377291
Author: Jaimin Jetly <ja...@hortonworks.com>
Authored: Thu Feb 11 15:02:09 2016 +0530
Committer: Jaimin Jetly <ja...@hortonworks.com>
Committed: Thu Feb 11 15:02:09 2016 +0530
----------------------------------------------------------------------
.../services/HIVE/configuration/hive-env.xml | 20 ---------
.../services/HIVE/configuration/hive-site.xml | 14 ------
.../stacks/HDP/2.2/services/stack_advisor.py | 13 +++---
.../services/HIVE/configuration/hive-env.xml | 20 ---------
.../services/HIVE/configuration/hive-site.xml | 8 ----
.../stacks/HDPWIN/2.2/services/stack_advisor.py | 13 +++---
.../2.1/configs/hive-metastore-upgrade.json | 3 +-
.../stacks/2.2/common/test_stack_advisor.py | 12 +++--
.../2.3/common/services-sparkts-hive.json | 47 --------------------
.../stacks/2.3/common/test_stack_advisor.py | 5 +--
10 files changed, 18 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8f3ee9d/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
index f84bc01..77eace4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml
@@ -127,26 +127,6 @@ export METASTORE_PORT={{hive_metastore_port}}
</property>
<property>
- <name>cost_based_optimizer</name>
- <display-name>Cost Based Optimizer</display-name>
- <value>Off</value>
- <value-attributes>
- <type>value-list</type>
- <entries>
- <entry>
- <value>On</value>
- <label>On</label>
- </entry>
- <entry>
- <value>Off</value>
- <label>Off</label>
- </entry>
- </entries>
- <selection-cardinality>1</selection-cardinality>
- </value-attributes>
- </property>
-
- <property>
<name>hive_security_authorization</name>
<display-name>Choose Authorization</display-name>
<value>None</value>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8f3ee9d/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
index 2076049..64094c0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
@@ -454,12 +454,6 @@ limitations under the License.
<name>hive.cbo.enable</name>
<value>true</value>
<description>Flag to control enabling Cost Based Optimizations using Calcite framework.</description>
- <depends-on>
- <property>
- <type>hive-env</type>
- <name>cost_based_optimizer</name>
- </property>
- </depends-on>
</property>
<property>
@@ -803,10 +797,6 @@ limitations under the License.
</value-attributes>
<depends-on>
<property>
- <type>hive-env</type>
- <name>cost_based_optimizer</name>
- </property>
- <property>
<type>hive-site</type>
<name>hive.cbo.enable</name>
</property>
@@ -838,10 +828,6 @@ limitations under the License.
</value-attributes>
<depends-on>
<property>
- <type>hive-env</type>
- <name>cost_based_optimizer</name>
- </property>
- <property>
<type>hive-site</type>
<name>hive.cbo.enable</name>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8f3ee9d/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 9be6410..3ebb25f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -347,14 +347,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864")
# CBO
- putHiveEnvProperty("cost_based_optimizer", "On")
- if str(configurations["hive-env"]["properties"]["cost_based_optimizer"]).lower() == "on":
- putHiveSiteProperty("hive.cbo.enable", "true")
- else:
- putHiveSiteProperty("hive.cbo.enable", "false")
- hive_cbo_enable = configurations["hive-site"]["properties"]["hive.cbo.enable"]
- putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable)
- putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable)
+ if "hive-site" in services["configurations"] and "hive.cbo.enable" in services["configurations"]["hive-site"]["properties"]:
+ hive_cbo_enable = services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"]
+ putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable)
+ putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable)
+
putHiveSiteProperty("hive.compute.query.using.stats", "true")
# Interactive Query
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8f3ee9d/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml
index 3d1b549..b5a9124 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml
@@ -74,26 +74,6 @@
</property>
<property>
- <name>cost_based_optimizer</name>
- <display-name>Cost Based Optimizer</display-name>
- <value>Off</value>
- <value-attributes>
- <type>value-list</type>
- <entries>
- <entry>
- <value>On</value>
- <label>On</label>
- </entry>
- <entry>
- <value>Off</value>
- <label>Off</label>
- </entry>
- </entries>
- <selection-cardinality>1</selection-cardinality>
- </value-attributes>
- </property>
-
- <property>
<name>hive_security_authorization</name>
<display-name>Choose Authorization</display-name>
<value>None</value>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8f3ee9d/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml
index 7e13436..8acc4dc 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml
@@ -772,10 +772,6 @@ limitations under the License.
</value-attributes>
<depends-on>
<property>
- <type>hive-env</type>
- <name>cost_based_optimizer</name>
- </property>
- <property>
<type>hive-site</type>
<name>hive.cbo.enable</name>
</property>
@@ -807,10 +803,6 @@ limitations under the License.
</value-attributes>
<depends-on>
<property>
- <type>hive-env</type>
- <name>cost_based_optimizer</name>
- </property>
- <property>
<type>hive-site</type>
<name>hive.cbo.enable</name>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8f3ee9d/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
index c6941f0..bf50aac 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
@@ -360,14 +360,11 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor):
putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864")
# CBO
- putHiveEnvProperty("cost_based_optimizer", "On")
- if str(configurations["hive-env"]["properties"]["cost_based_optimizer"]).lower() == "on":
- putHiveSiteProperty("hive.cbo.enable", "true")
- else:
- putHiveSiteProperty("hive.cbo.enable", "false")
- hive_cbo_enable = configurations["hive-site"]["properties"]["hive.cbo.enable"]
- putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable)
- putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable)
+ if "hive-site" in services["configurations"] and "hive.cbo.enable" in services["configurations"]["hive-site"]["properties"]:
+ hive_cbo_enable = services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"]
+ putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable)
+ putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable)
+
putHiveSiteProperty("hive.compute.query.using.stats", "true")
# Interactive Query
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8f3ee9d/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
index 7829f05..0fbc282 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json
@@ -77,8 +77,7 @@
"hive_user": "hive",
"hive_database": "New MySQL Database",
"hive.heapsize": "703",
- "hcat_pid_dir": "/var/run/webhcat",
- "cost_based_optimizer": "On",
+ "hcat_pid_dir": "/var/run/webhcat",
"webhcat_user": "hcat",
"hive_security_authorization": "None",
"hive_exec_orc_storage_strategy": "SPEED",
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8f3ee9d/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index ba470a1..6984e0e 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -983,7 +983,6 @@ class TestHDP22StackAdvisor(TestCase):
},
'hive-env': {
'properties': {
- 'cost_based_optimizer': 'On',
'hive_exec_orc_storage_strategy': 'SPEED',
'hive_security_authorization': 'None',
'hive_timeline_logging_enabled': 'true',
@@ -997,7 +996,6 @@ class TestHDP22StackAdvisor(TestCase):
'hive.server2.tez.initialize.default.sessions': 'false',
'hive.server2.tez.sessions.per.default.queue': '1',
'hive.auto.convert.join.noconditionaltask.size': '268435456',
- 'hive.cbo.enable': 'true',
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',
'hive.compute.query.using.stats': 'true',
@@ -1140,7 +1138,8 @@ class TestHDP22StackAdvisor(TestCase):
"hive.server2.authentication.kerberos.keytab": "",
"hive.server2.authentication.kerberos.principal": "",
"hive.server2.authentication.pam.services": "",
- "hive.server2.custom.authentication.class": ""
+ "hive.server2.custom.authentication.class": "",
+ "hive.cbo.enable": "true"
}
},
"hiveserver2-site": {
@@ -1215,7 +1214,8 @@ class TestHDP22StackAdvisor(TestCase):
"hive.server2.authentication.kerberos.keytab": "",
"hive.server2.authentication.kerberos.principal": "",
"hive.server2.authentication.pam.services": "",
- "hive.server2.custom.authentication.class": ""
+ "hive.server2.custom.authentication.class": "",
+ "hive.cbo.enable": "true"
}
},
"hiveserver2-site": {
@@ -1279,10 +1279,8 @@ class TestHDP22StackAdvisor(TestCase):
#test recommendations
services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"] = "false"
services["configurations"]["hive-env"]["properties"]["hive_security_authorization"] = "sqlstdauth"
- services["changed-configurations"] = [{"type": "hive-site", "name": "hive.cbo.enable"},
- {"type": "hive-env", "name": "hive_security_authorization"}]
+ services["changed-configurations"] = [{"type": "hive-env", "name": "hive_security_authorization"}]
expected["hive-env"]["properties"]["hive_security_authorization"] = "sqlstdauth"
- expected["hive-site"]["properties"]["hive.cbo.enable"] = "false"
expected["hive-site"]["properties"]["hive.stats.fetch.partition.stats"]="false"
expected["hive-site"]["properties"]["hive.stats.fetch.column.stats"]="false"
expected["hive-site"]["properties"]["hive.security.authorization.enabled"]="true"
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8f3ee9d/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json
index 51ae36a..20a672e 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json
+++ b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json
@@ -2402,47 +2402,6 @@
},
"dependencies" : [ ]
}, {
- "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer",
- "StackConfigurations" : {
- "property_depends_on" : [ ],
- "property_name" : "cost_based_optimizer",
- "service_name" : "HIVE",
- "stack_name" : "HDP",
- "stack_version" : "2.3",
- "type" : "hive-env.xml"
- },
- "dependencies" : [ {
- "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.cbo.enable",
- "StackConfigurationDependency" : {
- "dependency_name" : "hive.cbo.enable",
- "dependency_type" : "hive-site",
- "property_name" : "cost_based_optimizer",
- "service_name" : "HIVE",
- "stack_name" : "HDP",
- "stack_version" : "2.3"
- }
- }, {
- "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.stats.fetch.column.stats",
- "StackConfigurationDependency" : {
- "dependency_name" : "hive.stats.fetch.column.stats",
- "dependency_type" : "hive-site",
- "property_name" : "cost_based_optimizer",
- "service_name" : "HIVE",
- "stack_name" : "HDP",
- "stack_version" : "2.3"
- }
- }, {
- "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.stats.fetch.partition.stats",
- "StackConfigurationDependency" : {
- "dependency_name" : "hive.stats.fetch.partition.stats",
- "dependency_type" : "hive-site",
- "property_name" : "cost_based_optimizer",
- "service_name" : "HIVE",
- "stack_name" : "HDP",
- "stack_version" : "2.3"
- }
- } ]
- }, {
"href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/datanucleus.autoCreateSchema",
"StackConfigurations" : {
"property_depends_on" : [ ],
@@ -4199,9 +4158,6 @@
"href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/hive.stats.fetch.column.stats",
"StackConfigurations" : {
"property_depends_on" : [ {
- "type" : "hive-env",
- "name" : "cost_based_optimizer"
- }, {
"type" : "hive-site",
"name" : "hive.cbo.enable"
} ],
@@ -4216,9 +4172,6 @@
"href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/hive.stats.fetch.partition.stats",
"StackConfigurations" : {
"property_depends_on" : [ {
- "type" : "hive-env",
- "name" : "cost_based_optimizer"
- }, {
"type" : "hive-site",
"name" : "hive.cbo.enable"
} ],
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8f3ee9d/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 79f4f79..f0ad51b 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -870,7 +870,6 @@ class TestHDP23StackAdvisor(TestCase):
},
'hive-env': {
'properties': {
- 'cost_based_optimizer': 'On',
'hive_exec_orc_storage_strategy': 'SPEED',
'hive_security_authorization': 'None',
'hive_timeline_logging_enabled': 'true',
@@ -884,7 +883,6 @@ class TestHDP23StackAdvisor(TestCase):
'hive.server2.tez.initialize.default.sessions': 'false',
'hive.server2.tez.sessions.per.default.queue': '1',
'hive.auto.convert.join.noconditionaltask.size': '268435456',
- 'hive.cbo.enable': 'true',
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',
'hive.compute.query.using.stats': 'true',
@@ -1028,7 +1026,8 @@ class TestHDP23StackAdvisor(TestCase):
"hive.server2.authentication.kerberos.keytab": "",
"hive.server2.authentication.kerberos.principal": "",
"hive.server2.authentication.pam.services": "",
- "hive.server2.custom.authentication.class": ""
+ "hive.server2.custom.authentication.class": "",
+ "hive.cbo.enable": "true"
}
},
"hiveserver2-site": {