You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2017/10/11 23:04:04 UTC
[37/50] [abbrv] ambari git commit: Revert "AMBARI-22162. Move out the
druid configurations from hive-site to hive-interactive-site. (Slim Bouguerra
via Swapan Shridhar)."
Revert "AMBARI-22162. Move out the druid configurations from hive-site to hive-interactive-site. (Slim Bouguerra via Swapan Shridhar)."
This reverts commit ce2a0a00921bde8d780c82561902773f76431fce.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3b8e8071
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3b8e8071
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3b8e8071
Branch: refs/heads/branch-feature-AMBARI-14714-ui
Commit: 3b8e807100a988fafc4d55e7a0ad7644fac6cef2
Parents: 8cffd72
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Mon Oct 9 16:59:04 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Mon Oct 9 16:59:04 2017 -0400
----------------------------------------------------------------------
.../HIVE/0.12.0.2.0/configuration/hive-site.xml | 217 ++++++++++++++++++
.../configuration/hive-interactive-site.xml | 225 -------------------
.../stacks/HDP/2.6/services/stack_advisor.py | 12 +-
.../stacks/HDP/2.6/upgrades/config-upgrade.xml | 21 +-
4 files changed, 234 insertions(+), 241 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3b8e8071/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
index 69d1c69..d66cf4c 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
@@ -451,6 +451,223 @@ limitations under the License.
<on-ambari-upgrade add="false"/>
</property>
+ <!-- Druid related properties -->
+ <property>
+ <name>hive.druid.broker.address.default</name>
+ <value>localhost:8082</value>
+ <description>Host name of druid router if any or broker</description>
+ <on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>druid-router</type>
+ <name>druid.port</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>hive.druid.metadata.uri</name>
+ <value>jdbc:mysql://localhost:3355/druid</value>
+ <description>URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)</description>
+ <on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>druid-common</type>
+ <name>druid.metadata.storage.connector.connectURI</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>hive.druid.coordinator.address.default</name>
+ <value>localhost:8082</value>
+ <description>Host name of druid router if any or broker</description>
+ <on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>druid-coordinator</type>
+ <name>druid.port</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>hive.druid.metadata.password</name>
+ <value>{{druid_metadata_password}}</value>
+ <property-type>PASSWORD</property-type>
+ <display-name>Druid Metadata Password</display-name>
+ <description>Druid meta data storage password</description>
+ <value-attributes>
+ <type>password</type>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.metadata.username</name>
+ <value>druid</value>
+ <description>Username used to connect to druid metadata storage</description>
+ <on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>druid-common</type>
+ <name>druid.metadata.storage.connector.user</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>hive.druid.indexer.segments.granularity</name>
+ <display-name>Default Granularity for the Druid segments</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>YEAR</value>
+ </entry>
+ <entry>
+ <value>MONTH</value>
+ </entry>
+ <entry>
+ <value>WEEK</value>
+ </entry>
+ <entry>
+ <value>DAY</value>
+ </entry>
+ <entry>
+ <value>HOUR</value>
+ </entry>
+ <entry>
+ <value>MINUTE</value>
+ </entry>
+ <entry>
+ <value>SECOND</value>
+ </entry>
+ </entries>
+ </value-attributes>
+ <value>MINUTE</value>
+ <description>Default Granularity for the segments created by the Druid storage handler, this can be overridden per table using table property druid.segment.granularity </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.druid.indexer.partition.size.max</name>
+ <value>5000000</value>
+ <description>Maximum number of records per segment partition</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.druid.indexer.memory.rownum.max</name>
+ <value>75000</value>
+ <description>Maximum number of records in memory while storing data in Druid</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.select.distribute</name>
+ <value>true</value>
+ <description>If it is set to true, we distribute the execution of Druid Select queries</description>
+ <on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>hive.druid.basePersistDirectory</name>
+ <value></value>
+ <description>
+ Local temporary directory used to persist intermediate indexing state,
+ if empty (recommended) will default to JVM system property java.io.tmpdir.
+ </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.storage.storageDirectory</name>
+ <value>{{druid_storage_dir}}</value>
+ <description>
+ Druid deep storage location for segments.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.druid.metadata.db.type</name>
+ <display-name>Druid metadata storage type </display-name>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>mysql</value>
+ <label>MYSQL</label>
+ </entry>
+ <entry>
+ <value>postgresql</value>
+ <label>POSTGRESQL</label>
+ </entry>
+ </entries>
+ </value-attributes>
+ <value>mysql</value>
+ <depends-on>
+ <property>
+ <type>druid-common</type>
+ <name>druid.metadata.storage.type</name>
+ </property>
+ </depends-on>
+ <description>Druid metadata storage type</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.passiveWaitTimeMs</name>
+ <value>30000</value>
+ <description>
+ Wait time in ms default to 30 seconds.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.working.directory</name>
+ <value>/tmp/druid-indexing</value>
+ <description>
+ Default hdfs working directory used to store some intermediate metadata.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.maxTries</name>
+ <value>5</value>
+ <description>
+ Maximum number of http call retries before giving up.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.bitmap.type</name>
+ <display-name>Druid metadata storage type </display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>roaring</value>
+ </entry>
+ <entry>
+ <value>concise</value>
+ </entry>
+ </entries>
+ </value-attributes>
+ <value>roaring</value>
+ <description>Druid Coding algorithm use to encode the bitmaps</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
<!-- This property is removed in HDP 2.5 and higher. -->
<property>
<name>atlas.rest.address</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3b8e8071/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
index 64cef3e..aae2efa 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
@@ -124,229 +124,4 @@ limitations under the License.
<on-ambari-upgrade add="false"/>
</property>
- <!-- Druid related properties -->
- <property>
- <name>hive.druid.broker.address.default</name>
- <value>localhost:8082</value>
- <description>Host name of druid router if any or broker</description>
- <on-ambari-upgrade add="false"/>
- <depends-on>
- <property>
- <type>druid-router</type>
- <name>druid.port</name>
- </property>
- </depends-on>
- </property>
-
- <property>
- <name>hive.druid.metadata.uri</name>
- <value>jdbc:mysql://localhost:3355/druid</value>
- <description>URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)</description>
- <on-ambari-upgrade add="false"/>
- <depends-on>
- <property>
- <type>druid-common</type>
- <name>druid.metadata.storage.connector.connectURI</name>
- </property>
- </depends-on>
- </property>
-
- <property>
- <name>hive.druid.coordinator.address.default</name>
- <value>localhost:8082</value>
- <description>Host name of druid router if any or broker</description>
- <on-ambari-upgrade add="false"/>
- <depends-on>
- <property>
- <type>druid-coordinator</type>
- <name>druid.port</name>
- </property>
- </depends-on>
- </property>
-
- <property>
- <name>hive.druid.metadata.password</name>
- <value>{{druid_metadata_password}}</value>
- <property-type>PASSWORD</property-type>
- <display-name>Druid Metadata Password</display-name>
- <description>Druid meta data storage password</description>
- <value-attributes>
- <type>password</type>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.metadata.username</name>
- <value>druid</value>
- <description>Username used to connect to druid metadata storage</description>
- <on-ambari-upgrade add="false"/>
- <depends-on>
- <property>
- <type>druid-common</type>
- <name>druid.metadata.storage.connector.user</name>
- </property>
- </depends-on>
- </property>
-
- <property>
- <name>hive.druid.indexer.segments.granularity</name>
- <display-name>Default Granularity for the Druid segments</display-name>
- <value-attributes>
- <type>value-list</type>
- <entries>
- <entry>
- <value>YEAR</value>
- </entry>
- <entry>
- <value>MONTH</value>
- </entry>
- <entry>
- <value>WEEK</value>
- </entry>
- <entry>
- <value>DAY</value>
- </entry>
- <entry>
- <value>HOUR</value>
- </entry>
- <entry>
- <value>MINUTE</value>
- </entry>
- <entry>
- <value>SECOND</value>
- </entry>
- </entries>
- </value-attributes>
- <value>DAY</value>
- <description>Default Granularity for the segments created by the Druid storage handler, this can be overridden per table using table property druid.segment.granularity </description>
- <on-ambari-upgrade add="false"/>
- </property>
- <property>
- <name>hive.druid.indexer.partition.size.max</name>
- <value>1000000</value>
- <description>Maximum number of records per segment partition</description>
- <on-ambari-upgrade add="false"/>
- </property>
- <property>
- <name>hive.druid.indexer.memory.rownum.max</name>
- <value>75000</value>
- <description>Maximum number of records in memory while storing data in Druid</description>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.select.distribute</name>
- <value>true</value>
- <description>If it is set to true, we distribute the execution of Druid Select queries</description>
- <on-ambari-upgrade add="false"/>
- <value-attributes>
- <type>boolean</type>
- </value-attributes>
- </property>
-
- <property>
- <name>hive.druid.basePersistDirectory</name>
- <value></value>
- <description>
- Local temporary directory used to persist intermediate indexing state,
- if empty (recommended) will default to JVM system property java.io.tmpdir.
- </description>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.storage.storageDirectory</name>
- <value>{{druid_storage_dir}}</value>
- <description>
- Druid deep storage location for segments.
- </description>
- <on-ambari-upgrade add="false"/>
- </property>
- <property>
- <name>hive.druid.metadata.db.type</name>
- <display-name>Druid metadata storage type </display-name>
- <value-attributes>
- <overridable>false</overridable>
- <type>value-list</type>
- <entries>
- <entry>
- <value>mysql</value>
- <label>MYSQL</label>
- </entry>
- <entry>
- <value>postgresql</value>
- <label>POSTGRESQL</label>
- </entry>
- </entries>
- </value-attributes>
- <value>mysql</value>
- <depends-on>
- <property>
- <type>druid-common</type>
- <name>druid.metadata.storage.type</name>
- </property>
- </depends-on>
- <description>Druid metadata storage type</description>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.passiveWaitTimeMs</name>
- <value>30000</value>
- <description>
- Wait time in ms default to 30 seconds.
- </description>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.working.directory</name>
- <value>/tmp/druid-indexing</value>
- <description>
- Default hdfs working directory used to store some intermediate metadata.
- </description>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.maxTries</name>
- <value>5</value>
- <description>
- Maximum number of http call retries before giving up.
- </description>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.bitmap.type</name>
- <display-name>Druid metadata storage type </display-name>
- <value-attributes>
- <type>value-list</type>
- <entries>
- <entry>
- <value>roaring</value>
- </entry>
- <entry>
- <value>concise</value>
- </entry>
- </entries>
- </value-attributes>
- <value>roaring</value>
- <description>Druid Coding algorithm use to encode the bitmaps</description>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.http.read.timeout</name>
- <value>PT10M</value>
- <description>
- Maximum number of http call retries before giving up.
- </description>
- <on-ambari-upgrade add="false"/>
- </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3b8e8071/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index b634e71..0d2925e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -605,7 +605,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
# druid is not in list of services to be installed
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
if 'DRUID' in servicesList:
- putHiveInteractiveSiteProperty = self.putProperty(configurations, "hive-interactive-site", services)
+ putHiveSiteProperty = self.putProperty(configurations, "hive-site", services)
if 'druid-coordinator' in services['configurations']:
component_hosts = self.getHostsWithComponent("DRUID", 'DRUID_COORDINATOR', services, hosts)
if component_hosts is not None and len(component_hosts) > 0:
@@ -642,11 +642,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
else:
druid_metadata_user = ""
- putHiveInteractiveSiteProperty('hive.druid.broker.address.default', druid_broker_host_port)
- putHiveInteractiveSiteProperty('hive.druid.coordinator.address.default', druid_coordinator_host_port)
- putHiveInteractiveSiteProperty('hive.druid.metadata.uri', druid_metadata_uri)
- putHiveInteractiveSiteProperty('hive.druid.metadata.username', druid_metadata_user)
- putHiveInteractiveSiteProperty('hive.druid.metadata.db.type', druid_metadata_type)
+ putHiveSiteProperty('hive.druid.broker.address.default', druid_broker_host_port)
+ putHiveSiteProperty('hive.druid.coordinator.address.default', druid_coordinator_host_port)
+ putHiveSiteProperty('hive.druid.metadata.uri', druid_metadata_uri)
+ putHiveSiteProperty('hive.druid.metadata.username', druid_metadata_user)
+ putHiveSiteProperty('hive.druid.metadata.db.type', druid_metadata_type)
def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
http://git-wip-us.apache.org/repos/asf/ambari/blob/3b8e8071/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
index fd7e438..2b4c656 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
@@ -75,19 +75,11 @@
<set key ="atlas.jaas.ticketBased-KafkaClient.option.useTicketCache" value="true"
if-type="cluster-env" if-key="security_enabled" if-value="true"/>
</definition>
- </changes>
- </component>
- <component name="HIVE_SERVER_INTERACTIVE">
- <changes>
- <definition xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable" summary="Update additional LLAP-Tez settings">
- <type>tez-interactive-site</type>
- <set key="tez.runtime.shuffle.ssl.enable" value="false"/>
- </definition>
<definition xsi:type="configure" id="hdp_2_6_maint_druid_config_for_hive_hook" summary="Updating druid hive related properties">
<type>hive-site</type>
<set key="hive.druid.metadata.password" value="{{druid_metadata_password}}" if-type="druid-common"/>
- <set key="hive.druid.indexer.segments.granularity" value="DAY" if-type="druid-common"/>
- <set key="hive.druid.indexer.partition.size.max" value="1000000" if-type="druid-common"/>
+ <set key="hive.druid.indexer.segments.granularity" value="MINUTE" if-type="druid-common"/>
+ <set key="hive.druid.indexer.partition.size.max" value="5000000" if-type="druid-common"/>
<set key="hive.druid.indexer.memory.rownum.max" value="75000" if-type="druid-common"/>
<set key="hive.druid.select.distribute" value="true" if-type="druid-common"/>
<set key="hive.druid.basePersistDirectory" value="" if-type="druid-common"/>
@@ -95,6 +87,15 @@
<set key="hive.druid.passiveWaitTimeMs" value="30000" if-type="druid-common"/>
<set key="hive.druid.working.directory" value="/tmp/druid-indexing" if-type="druid-common"/>
<set key="hive.druid.bitmap.type" value="roaring" if-type="druid-common"/>
+
+ </definition>
+ </changes>
+ </component>
+ <component name="HIVE_SERVER_INTERACTIVE">
+ <changes>
+ <definition xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable" summary="Update additional LLAP-Tez settings">
+ <type>tez-interactive-site</type>
+ <set key="tez.runtime.shuffle.ssl.enable" value="false"/>
</definition>
</changes>
</component>