You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/02/21 16:39:45 UTC
[14/25] ambari git commit: AMBARI-20069.
hive.enforce.bucketing=true.(vbrodetskyi)
AMBARI-20069. hive.enforce.bucketing=true.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/60aaaeab
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/60aaaeab
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/60aaaeab
Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 60aaaeab5e2e9c7581f982525950bf6290db3018
Parents: 1ecdee0
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Tue Feb 21 02:15:11 2017 +0200
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Tue Feb 21 02:15:11 2017 +0200
----------------------------------------------------------------------
.../main/resources/stacks/HDP/2.2/services/stack_advisor.py | 7 +++++--
.../main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml | 5 +++++
.../stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 6 ++++++
.../main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 2 ++
.../main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml | 5 +++++
.../stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 6 ++++++
.../main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 2 ++
.../main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml | 5 +++++
.../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 6 ++++++
.../main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 1 +
.../src/test/python/stacks/2.2/common/test_stack_advisor.py | 1 -
.../src/test/python/stacks/2.3/common/test_stack_advisor.py | 2 --
12 files changed, 43 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 0d7b8b9..ede41fc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -374,14 +374,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
putHiveSiteProperty("hive.support.concurrency", "true")
putHiveSiteProperty("hive.compactor.initiator.on", "true")
putHiveSiteProperty("hive.compactor.worker.threads", "1")
- putHiveSiteProperty("hive.enforce.bucketing", "true")
putHiveSiteProperty("hive.exec.dynamic.partition.mode", "nonstrict")
else:
putHiveSiteProperty("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager")
putHiveSiteProperty("hive.support.concurrency", "false")
putHiveSiteProperty("hive.compactor.initiator.on", "false")
putHiveSiteProperty("hive.compactor.worker.threads", "0")
- putHiveSiteProperty("hive.enforce.bucketing", "false")
putHiveSiteProperty("hive.exec.dynamic.partition.mode", "strict")
hiveMetastoreHost = self.getHostWithComponent("HIVE", "HIVE_METASTORE", services, hosts)
@@ -1497,6 +1495,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
ldap_domain_property + " property, if you are using AD, if not, then " + ldap_baseDN_property + "!")})
+ hive_enforce_bucketing = "hive.enforce.bucketing"
+ if hive_enforce_bucketing in properties and properties[hive_enforce_bucketing].lower() == "false":
+ validationItems.append({"config-name" : hive_enforce_bucketing, "item" :
+ self.getWarnItem("Set " + hive_enforce_bucketing + " to true otherwise there is a potential of data corruption!")})
+
configurationValidationProblems = self.toConfigurationValidationProblems(validationItems, "hive-site")
configurationValidationProblems.extend(parentValidationProblems)
return configurationValidationProblems
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index ba155ab..cdd701a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -195,6 +195,11 @@
<regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/>
<replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/>
</definition>
+
+ <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <type>hive-site</type>
+ <set key="hive.enforce.bucketing" value="true"/>
+ </definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
index e6a3c94..ff42022 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
@@ -419,6 +419,12 @@
</task>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing property">
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <summary>Updating hive.enforce.bucketing property to true value</summary>
+ </task>
+ </execute-stage>
+
<!--OOZIE-->
<execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes for Oozie Server">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation">
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
index 7d8438f..b255bc0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
@@ -893,6 +893,8 @@
<summary>Update hive-env content</summary>
</task>
<task xsi:type="configure" id="hive_log4j_parameterize" />
+
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property" />
</pre-upgrade>
<pre-downgrade/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index d19bb76..1bea263 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -96,6 +96,11 @@
<regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)" replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/>
<replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender" replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB"/>
</definition>
+
+ <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <type>hive-site</type>
+ <set key="hive.enforce.bucketing" value="true"/>
+ </definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
index 69a894c..86cd56a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
@@ -461,6 +461,12 @@
</task>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing property">
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <summary>Updating hive.enforce.bucketing property to true value</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat Log4J Properties">
<task xsi:type="configure" id="webhcat_log4j_parameterize">
<summary>Updating the Webhcat Log4J properties to include parameterizations</summary>
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
index 0c9a8ea..6a8e9d7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
@@ -892,6 +892,8 @@
</task>
<task xsi:type="configure" id="hive_log4j_parameterize" />
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property" />
+
</pre-upgrade>
<pre-downgrade/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 241ca8f..ca9cf47 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -358,6 +358,11 @@
<regex-replace key="content" find="property.llap.daemon.log.maxfilesize = ([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/>
<regex-replace key="content" find="property.llap.daemon.log.maxbackupindex = ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/>
</definition>
+
+ <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <type>hive-site</type>
+ <set key="hive.enforce.bucketing" value="true"/>
+ </definition>
</changes>
</component>
<component name="HIVE_SERVER_INTERACTIVE">
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index 0ef69cb..06cae78 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -478,6 +478,12 @@
<task xsi:type="configure" id="hdp_2_6_0_0_tez_append_heap_dump_options_for_tez_am"/>
</execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing property">
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+ <summary>Updating hive.enforce.bucketing property to true value</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Appending heap dump options for HiveSever2 Interactive">
<task xsi:type="configure" id="hdp_2_6_0_0_hive_llap_append_heap_dump_options"/>
</execute-stage>
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index 6309bcb..64edbb8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -781,6 +781,7 @@
<task xsi:type="configure" id="hdp_2_6_0_0_hive_append_heap_dump_options"/>
<task xsi:type="configure" id="hive_log4j_parameterize" />
<task xsi:type="configure" id="hive_llap_log4j_parameterize" />
+ <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property" />
</pre-upgrade>
<pre-downgrade />
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 0c7af1a..b8762b3 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1300,7 +1300,6 @@ class TestHDP22StackAdvisor(TestCase):
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',
'hive.compute.query.using.stats': 'true',
- 'hive.enforce.bucketing': 'false',
'hive.exec.dynamic.partition.mode': 'strict',
'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
'hive.exec.orc.compression.strategy': 'SPEED',
http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 7e93815..443331a 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -605,7 +605,6 @@ class TestHDP23StackAdvisor(TestCase):
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',
'hive.compute.query.using.stats': 'true',
- 'hive.enforce.bucketing': 'false',
'hive.exec.dynamic.partition.mode': 'strict',
'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
'hive.exec.orc.compression.strategy': 'SPEED',
@@ -883,7 +882,6 @@ class TestHDP23StackAdvisor(TestCase):
'hive.compactor.initiator.on': 'false',
'hive.compactor.worker.threads': '0',
'hive.compute.query.using.stats': 'true',
- 'hive.enforce.bucketing': 'false',
'hive.exec.dynamic.partition.mode': 'strict',
'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
'hive.exec.orc.compression.strategy': 'SPEED',