You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2017/02/21 01:01:09 UTC
ambari git commit: AMBARI-20069.
hive.enforce.bucketing=true.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 1169694ec -> 5e8f9dc3d
AMBARI-20069. hive.enforce.bucketing=true.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5e8f9dc3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5e8f9dc3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5e8f9dc3
Branch: refs/heads/branch-2.5
Commit: 5e8f9dc3d47f15d157f24dfc67c47b7488b5535f
Parents: 1169694
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Tue Feb 21 02:03:46 2017 +0200
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Tue Feb 21 02:03:46 2017 +0200
----------------------------------------------------------------------
.../main/resources/stacks/HDP/2.2/services/stack_advisor.py | 7 +++++--
.../main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml | 5 +++++
.../stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 6 ++++++
.../main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 2 ++
.../main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml | 5 +++++
.../stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 6 ++++++
.../main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 2 ++
.../main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml | 5 +++++
.../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 6 ++++++
.../main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 1 +
.../src/test/python/stacks/2.2/common/test_stack_advisor.py | 1 -
.../src/test/python/stacks/2.3/common/test_stack_advisor.py | 2 --
12 files changed, 43 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 5e0fa5e..f4583ba 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -320,14 +320,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHiveSiteProperty("hive.support.concurrency", "true")
putHiveSiteProperty("hive.compactor.initiator.on", "true")
putHiveSiteProperty("hive.compactor.worker.threads", "1")
- putHiveSiteProperty("hive.enforce.bucketing", "true")
putHiveSiteProperty("hive.exec.dynamic.partition.mode", "nonstrict")
else:
putHiveSiteProperty("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager")
putHiveSiteProperty("hive.support.concurrency", "false")
putHiveSiteProperty("hive.compactor.initiator.on", "false")
putHiveSiteProperty("hive.compactor.worker.threads", "0")
- putHiveSiteProperty("hive.enforce.bucketing", "false")
putHiveSiteProperty("hive.exec.dynamic.partition.mode", "strict")
hiveMetastoreHost = self.getHostWithComponent("HIVE", "HIVE_METASTORE", services, hosts)
@@ -1443,6 +1441,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
ldap_domain_property + " property, if you are using AD, if not, then " + ldap_baseDN_property + "!")})
+ hive_enforce_bucketing = "hive.enforce.bucketing"
+ if hive_enforce_bucketing in properties and properties[hive_enforce_bucketing].lower() == "false":
+ validationItems.append({"config-name" : hive_enforce_bucketing, "item" :
+ self.getWarnItem("Set " + hive_enforce_bucketing + " to true otherwise there is a potential of data corruption!")})
+
configurationValidationProblems = self.toConfigurationValidationProblems(validationItems, "hive-site")
configurationValidationProblems.extend(parentValidationProblems)
return configurationValidationProblems
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index 0e48fdf..84e79a2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -195,6 +195,11 @@
<regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/>
<replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/>
</definition>
+
+ <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <type>hive-site</type>
+ <set key="hive.enforce.bucketing" value="true"/>
+ </definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
index b4205d9..8f84ed1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
@@ -419,6 +419,12 @@
</task>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing property">
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <summary>Updating hive.enforce.bucketing property to true value</summary>
+ </task>
+ </execute-stage>
+
<!--OOZIE-->
<execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation">
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
index 7d8438f..b255bc0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
@@ -893,6 +893,8 @@
<summary>Update hive-env content</summary>
</task>
<task xsi:type="configure" id="hive_log4j_parameterize" />
+
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property" />
</pre-upgrade>
<pre-downgrade/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index 6da6c22..2a60a04 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -96,6 +96,11 @@
<regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/>
<replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/>
</definition>
+
+ <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <type>hive-site</type>
+ <set key="hive.enforce.bucketing" value="true"/>
+ </definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
index 437ca42..f1aedf5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
@@ -461,6 +461,12 @@
</task>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing property">
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <summary>Updating hive.enforce.bucketing property to true value</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties">
<task xsi:type="configure" id="webhcat_log4j_parameterize">
<summary>Updating the Webhcat Log4J properties to include parameterizations</summary>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
index 3abaee3..a16003c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
@@ -892,6 +892,8 @@
</task>
<task xsi:type="configure" id="hive_log4j_parameterize" />
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property" />
+
</pre-upgrade>
<pre-downgrade/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 31bb296..e9cd198 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -351,6 +351,11 @@
<regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/>
<regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/>
</definition>
+
+ <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <type>hive-site</type>
+ <set key="hive.enforce.bucketing" value="true"/>
+ </definition>
</changes>
</component>
<component name="HIVE_SERVER_INTERACTIVE">
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index 09c2b06..64460be 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -474,6 +474,12 @@
<task xsi:type="configure" id="hdp_2_6_0_0_tez_append_heap_dump_options_for_tez_am"/>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing property">
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <summary>Updating hive.enforce.bucketing property to true value</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Appending heap dump options for HiveSever2 Interactive">
<task xsi:type="configure" id="hdp_2_6_0_0_hive_llap_append_heap_dump_options"/>
</execute-stage>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index 6309bcb..64edbb8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -781,6 +781,7 @@
<task xsi:type="configure" id="hdp_2_6_0_0_hive_append_heap_dump_options"/>
<task xsi:type="configure" id="hive_log4j_parameterize" />
<task xsi:type="configure" id="hive_llap_log4j_parameterize" />
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property" />
</pre-upgrade>
<pre-downgrade />
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index e2813fa..9c3fff6 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1300,7 +1300,6 @@ class TestHDP22StackAdvisor(TestCase):
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',
'hive.compute.query.using.stats': 'true',
- 'hive.enforce.bucketing': 'false',
'hive.exec.dynamic.partition.mode': 'strict',
'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
'hive.exec.orc.compression.strategy': 'SPEED',
http://git-wip-us.apache.org/repos/asf/ambari/blob/5e8f9dc3/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index b2438d9..b1ec05b 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -604,7 +604,6 @@ class TestHDP23StackAdvisor(TestCase):
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',
'hive.compute.query.using.stats': 'true',
- 'hive.enforce.bucketing': 'false',
'hive.exec.dynamic.partition.mode': 'strict',
'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
'hive.exec.orc.compression.strategy': 'SPEED',
@@ -882,7 +881,6 @@ class TestHDP23StackAdvisor(TestCase):
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',
'hive.compute.query.using.stats': 'true',
- 'hive.enforce.bucketing': 'false',
'hive.exec.dynamic.partition.mode': 'strict',
'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
'hive.exec.orc.compression.strategy': 'SPEED',