You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2017/10/13 15:10:42 UTC
[23/50] [abbrv] ambari git commit: AMBARI-22162. Move out the druid
configurations from hive-site to hive-interactive-site. (Slim Bouguerra via
Swapan Shridhar).
AMBARI-22162. Move out the druid configurations from hive-site to hive-interactive-site. (Slim Bouguerra via Swapan Shridhar).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7e885a39
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7e885a39
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7e885a39
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: 7e885a390888a5cddc4cf448af3cfd225f59cf3d
Parents: 03273bd
Author: Swapan Shridhar <ss...@hortonworks.com>
Authored: Wed Oct 11 00:57:01 2017 -0700
Committer: Swapan Shridhar <ss...@hortonworks.com>
Committed: Wed Oct 11 00:57:13 2017 -0700
----------------------------------------------------------------------
.../HIVE/0.12.0.2.0/configuration/hive-site.xml | 217 ------------------
.../configuration/hive-interactive-site.xml | 225 +++++++++++++++++++
.../stacks/HDP/2.6/services/stack_advisor.py | 13 +-
.../stacks/HDP/2.6/upgrades/config-upgrade.xml | 21 +-
.../stacks/2.6/common/test_stack_advisor.py | 17 +-
5 files changed, 250 insertions(+), 243 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e885a39/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
index d66cf4c..69d1c69 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
@@ -451,223 +451,6 @@ limitations under the License.
<on-ambari-upgrade add="false"/>
</property>
- <!-- Druid related properties -->
- <property>
- <name>hive.druid.broker.address.default</name>
- <value>localhost:8082</value>
- <description>Host name of druid router if any or broker</description>
- <on-ambari-upgrade add="false"/>
- <depends-on>
- <property>
- <type>druid-router</type>
- <name>druid.port</name>
- </property>
- </depends-on>
- </property>
-
- <property>
- <name>hive.druid.metadata.uri</name>
- <value>jdbc:mysql://localhost:3355/druid</value>
- <description>URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)</description>
- <on-ambari-upgrade add="false"/>
- <depends-on>
- <property>
- <type>druid-common</type>
- <name>druid.metadata.storage.connector.connectURI</name>
- </property>
- </depends-on>
- </property>
-
- <property>
- <name>hive.druid.coordinator.address.default</name>
- <value>localhost:8082</value>
- <description>Host name of druid router if any or broker</description>
- <on-ambari-upgrade add="false"/>
- <depends-on>
- <property>
- <type>druid-coordinator</type>
- <name>druid.port</name>
- </property>
- </depends-on>
- </property>
-
- <property>
- <name>hive.druid.metadata.password</name>
- <value>{{druid_metadata_password}}</value>
- <property-type>PASSWORD</property-type>
- <display-name>Druid Metadata Password</display-name>
- <description>Druid meta data storage password</description>
- <value-attributes>
- <type>password</type>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.metadata.username</name>
- <value>druid</value>
- <description>Username used to connect to druid metadata storage</description>
- <on-ambari-upgrade add="false"/>
- <depends-on>
- <property>
- <type>druid-common</type>
- <name>druid.metadata.storage.connector.user</name>
- </property>
- </depends-on>
- </property>
-
- <property>
- <name>hive.druid.indexer.segments.granularity</name>
- <display-name>Default Granularity for the Druid segments</display-name>
- <value-attributes>
- <type>value-list</type>
- <entries>
- <entry>
- <value>YEAR</value>
- </entry>
- <entry>
- <value>MONTH</value>
- </entry>
- <entry>
- <value>WEEK</value>
- </entry>
- <entry>
- <value>DAY</value>
- </entry>
- <entry>
- <value>HOUR</value>
- </entry>
- <entry>
- <value>MINUTE</value>
- </entry>
- <entry>
- <value>SECOND</value>
- </entry>
- </entries>
- </value-attributes>
- <value>MINUTE</value>
- <description>Default Granularity for the segments created by the Druid storage handler, this can be overridden per table using table property druid.segment.granularity </description>
- <on-ambari-upgrade add="false"/>
- </property>
- <property>
- <name>hive.druid.indexer.partition.size.max</name>
- <value>5000000</value>
- <description>Maximum number of records per segment partition</description>
- <on-ambari-upgrade add="false"/>
- </property>
- <property>
- <name>hive.druid.indexer.memory.rownum.max</name>
- <value>75000</value>
- <description>Maximum number of records in memory while storing data in Druid</description>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.select.distribute</name>
- <value>true</value>
- <description>If it is set to true, we distribute the execution of Druid Select queries</description>
- <on-ambari-upgrade add="false"/>
- <value-attributes>
- <type>boolean</type>
- </value-attributes>
- </property>
-
- <property>
- <name>hive.druid.basePersistDirectory</name>
- <value></value>
- <description>
- Local temporary directory used to persist intermediate indexing state,
- if empty (recommended) will default to JVM system property java.io.tmpdir.
- </description>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.storage.storageDirectory</name>
- <value>{{druid_storage_dir}}</value>
- <description>
- Druid deep storage location for segments.
- </description>
- <on-ambari-upgrade add="false"/>
- </property>
- <property>
- <name>hive.druid.metadata.db.type</name>
- <display-name>Druid metadata storage type </display-name>
- <value-attributes>
- <overridable>false</overridable>
- <type>value-list</type>
- <entries>
- <entry>
- <value>mysql</value>
- <label>MYSQL</label>
- </entry>
- <entry>
- <value>postgresql</value>
- <label>POSTGRESQL</label>
- </entry>
- </entries>
- </value-attributes>
- <value>mysql</value>
- <depends-on>
- <property>
- <type>druid-common</type>
- <name>druid.metadata.storage.type</name>
- </property>
- </depends-on>
- <description>Druid metadata storage type</description>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.passiveWaitTimeMs</name>
- <value>30000</value>
- <description>
- Wait time in ms default to 30 seconds.
- </description>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.working.directory</name>
- <value>/tmp/druid-indexing</value>
- <description>
- Default hdfs working directory used to store some intermediate metadata.
- </description>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.maxTries</name>
- <value>5</value>
- <description>
- Maximum number of http call retries before giving up.
- </description>
- <on-ambari-upgrade add="false"/>
- </property>
-
- <property>
- <name>hive.druid.bitmap.type</name>
- <display-name>Druid metadata storage type </display-name>
- <value-attributes>
- <type>value-list</type>
- <entries>
- <entry>
- <value>roaring</value>
- </entry>
- <entry>
- <value>concise</value>
- </entry>
- </entries>
- </value-attributes>
- <value>roaring</value>
- <description>Druid Coding algorithm use to encode the bitmaps</description>
- <on-ambari-upgrade add="false"/>
- </property>
-
<!-- This property is removed in HDP 2.5 and higher. -->
<property>
<name>atlas.rest.address</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e885a39/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
index aae2efa..64cef3e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
@@ -124,4 +124,229 @@ limitations under the License.
<on-ambari-upgrade add="false"/>
</property>
+ <!-- Druid related properties -->
+ <property>
+ <name>hive.druid.broker.address.default</name>
+ <value>localhost:8082</value>
+ <description>Host name of druid router if any or broker</description>
+ <on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>druid-router</type>
+ <name>druid.port</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>hive.druid.metadata.uri</name>
+ <value>jdbc:mysql://localhost:3355/druid</value>
+ <description>URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)</description>
+ <on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>druid-common</type>
+ <name>druid.metadata.storage.connector.connectURI</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>hive.druid.coordinator.address.default</name>
+ <value>localhost:8082</value>
+ <description>Host name of druid router if any or broker</description>
+ <on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>druid-coordinator</type>
+ <name>druid.port</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>hive.druid.metadata.password</name>
+ <value>{{druid_metadata_password}}</value>
+ <property-type>PASSWORD</property-type>
+ <display-name>Druid Metadata Password</display-name>
+ <description>Druid meta data storage password</description>
+ <value-attributes>
+ <type>password</type>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.metadata.username</name>
+ <value>druid</value>
+ <description>Username used to connect to druid metadata storage</description>
+ <on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>druid-common</type>
+ <name>druid.metadata.storage.connector.user</name>
+ </property>
+ </depends-on>
+ </property>
+
+ <property>
+ <name>hive.druid.indexer.segments.granularity</name>
+ <display-name>Default Granularity for the Druid segments</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>YEAR</value>
+ </entry>
+ <entry>
+ <value>MONTH</value>
+ </entry>
+ <entry>
+ <value>WEEK</value>
+ </entry>
+ <entry>
+ <value>DAY</value>
+ </entry>
+ <entry>
+ <value>HOUR</value>
+ </entry>
+ <entry>
+ <value>MINUTE</value>
+ </entry>
+ <entry>
+ <value>SECOND</value>
+ </entry>
+ </entries>
+ </value-attributes>
+ <value>DAY</value>
+ <description>Default Granularity for the segments created by the Druid storage handler, this can be overridden per table using table property druid.segment.granularity </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.druid.indexer.partition.size.max</name>
+ <value>1000000</value>
+ <description>Maximum number of records per segment partition</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.druid.indexer.memory.rownum.max</name>
+ <value>75000</value>
+ <description>Maximum number of records in memory while storing data in Druid</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.select.distribute</name>
+ <value>true</value>
+ <description>If it is set to true, we distribute the execution of Druid Select queries</description>
+ <on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>hive.druid.basePersistDirectory</name>
+ <value></value>
+ <description>
+ Local temporary directory used to persist intermediate indexing state,
+ if empty (recommended) will default to JVM system property java.io.tmpdir.
+ </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.storage.storageDirectory</name>
+ <value>{{druid_storage_dir}}</value>
+ <description>
+ Druid deep storage location for segments.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hive.druid.metadata.db.type</name>
+ <display-name>Druid metadata storage type </display-name>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>mysql</value>
+ <label>MYSQL</label>
+ </entry>
+ <entry>
+ <value>postgresql</value>
+ <label>POSTGRESQL</label>
+ </entry>
+ </entries>
+ </value-attributes>
+ <value>mysql</value>
+ <depends-on>
+ <property>
+ <type>druid-common</type>
+ <name>druid.metadata.storage.type</name>
+ </property>
+ </depends-on>
+ <description>Druid metadata storage type</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.passiveWaitTimeMs</name>
+ <value>30000</value>
+ <description>
+ Wait time in ms default to 30 seconds.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.working.directory</name>
+ <value>/tmp/druid-indexing</value>
+ <description>
+ Default hdfs working directory used to store some intermediate metadata.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.maxTries</name>
+ <value>5</value>
+ <description>
+ Maximum number of http call retries before giving up.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.bitmap.type</name>
+ <display-name>Druid metadata storage type </display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>roaring</value>
+ </entry>
+ <entry>
+ <value>concise</value>
+ </entry>
+ </entries>
+ </value-attributes>
+ <value>roaring</value>
+ <description>Druid Coding algorithm use to encode the bitmaps</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hive.druid.http.read.timeout</name>
+ <value>PT10M</value>
+ <description>
+ Maximum number of http call retries before giving up.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e885a39/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 0d2925e..38af525 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -605,7 +605,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
# druid is not in list of services to be installed
servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
if 'DRUID' in servicesList:
- putHiveSiteProperty = self.putProperty(configurations, "hive-site", services)
+ putHiveInteractiveSiteProperty = self.putProperty(configurations, "hive-interactive-site", services)
if 'druid-coordinator' in services['configurations']:
component_hosts = self.getHostsWithComponent("DRUID", 'DRUID_COORDINATOR', services, hosts)
if component_hosts is not None and len(component_hosts) > 0:
@@ -621,7 +621,6 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
if component_hosts is not None and len(component_hosts) > 0:
# pick the first
host = component_hosts[0]
- print host
druid_broker_host_port = str(host['Hosts']['host_name']) + ":" + str(
services['configurations']['druid-router']['properties']['druid.port'])
elif 'druid-broker' in services['configurations']:
@@ -642,11 +641,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
else:
druid_metadata_user = ""
- putHiveSiteProperty('hive.druid.broker.address.default', druid_broker_host_port)
- putHiveSiteProperty('hive.druid.coordinator.address.default', druid_coordinator_host_port)
- putHiveSiteProperty('hive.druid.metadata.uri', druid_metadata_uri)
- putHiveSiteProperty('hive.druid.metadata.username', druid_metadata_user)
- putHiveSiteProperty('hive.druid.metadata.db.type', druid_metadata_type)
+ putHiveInteractiveSiteProperty('hive.druid.broker.address.default', druid_broker_host_port)
+ putHiveInteractiveSiteProperty('hive.druid.coordinator.address.default', druid_coordinator_host_port)
+ putHiveInteractiveSiteProperty('hive.druid.metadata.uri', druid_metadata_uri)
+ putHiveInteractiveSiteProperty('hive.druid.metadata.username', druid_metadata_user)
+ putHiveInteractiveSiteProperty('hive.druid.metadata.db.type', druid_metadata_type)
def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e885a39/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
index 2b4c656..fd7e438 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
@@ -75,11 +75,19 @@
<set key ="atlas.jaas.ticketBased-KafkaClient.option.useTicketCache" value="true"
if-type="cluster-env" if-key="security_enabled" if-value="true"/>
</definition>
+ </changes>
+ </component>
+ <component name="HIVE_SERVER_INTERACTIVE">
+ <changes>
+ <definition xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable" summary="Update additional LLAP-Tez settings">
+ <type>tez-interactive-site</type>
+ <set key="tez.runtime.shuffle.ssl.enable" value="false"/>
+ </definition>
<definition xsi:type="configure" id="hdp_2_6_maint_druid_config_for_hive_hook" summary="Updating druid hive related properties">
<type>hive-site</type>
<set key="hive.druid.metadata.password" value="{{druid_metadata_password}}" if-type="druid-common"/>
- <set key="hive.druid.indexer.segments.granularity" value="MINUTE" if-type="druid-common"/>
- <set key="hive.druid.indexer.partition.size.max" value="5000000" if-type="druid-common"/>
+ <set key="hive.druid.indexer.segments.granularity" value="DAY" if-type="druid-common"/>
+ <set key="hive.druid.indexer.partition.size.max" value="1000000" if-type="druid-common"/>
<set key="hive.druid.indexer.memory.rownum.max" value="75000" if-type="druid-common"/>
<set key="hive.druid.select.distribute" value="true" if-type="druid-common"/>
<set key="hive.druid.basePersistDirectory" value="" if-type="druid-common"/>
@@ -87,15 +95,6 @@
<set key="hive.druid.passiveWaitTimeMs" value="30000" if-type="druid-common"/>
<set key="hive.druid.working.directory" value="/tmp/druid-indexing" if-type="druid-common"/>
<set key="hive.druid.bitmap.type" value="roaring" if-type="druid-common"/>
-
- </definition>
- </changes>
- </component>
- <component name="HIVE_SERVER_INTERACTIVE">
- <changes>
- <definition xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable" summary="Update additional LLAP-Tez settings">
- <type>tez-interactive-site</type>
- <set key="tez.runtime.shuffle.ssl.enable" value="false"/>
</definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e885a39/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
index ec44b3d..1c937b7 100644
--- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
@@ -1835,12 +1835,7 @@ class TestHDP26StackAdvisor(TestCase):
'hive.security.metastore.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider',
'hive.exec.dynamic.partition.mode': 'strict',
'hive.optimize.sort.dynamic.partition': 'false',
- 'hive.server2.enable.doAs': 'false',
- 'hive.druid.broker.address.default': 'c6401.ambari.apache.org:8082',
- 'hive.druid.coordinator.address.default': 'c6401.ambari.apache.org:8081',
- 'hive.druid.metadata.db.type': 'mysql',
- 'hive.druid.metadata.uri': 'jdbc:mysql://c6401.ambari.apache.org:3306/druid?createDatabaseIfNotExist=true',
- 'hive.druid.metadata.username': 'druid',
+ 'hive.server2.enable.doAs': 'false'
},
'property_attributes': {
'hive.tez.container.size': {
@@ -1885,7 +1880,13 @@ class TestHDP26StackAdvisor(TestCase):
}
},
'hive-interactive-site': {
- 'properties': {}
+ 'properties': {
+ 'hive.druid.broker.address.default': 'c6401.ambari.apache.org:8082',
+ 'hive.druid.coordinator.address.default': 'c6401.ambari.apache.org:8081',
+ 'hive.druid.metadata.db.type': 'mysql',
+ 'hive.druid.metadata.uri': 'jdbc:mysql://c6401.ambari.apache.org:3306/druid?createDatabaseIfNotExist=true',
+ 'hive.druid.metadata.username': 'druid'
+ }
},
'yarn-site': {
'properties': {
@@ -1941,7 +1942,7 @@ class TestHDP26StackAdvisor(TestCase):
services['configurations']['druid-router'] = {}
services['configurations']['druid-router']['properties'] = {}
services['configurations']['druid-router']['properties']['druid.port'] = 8083
- expected['hive-site']['properties']['hive.druid.broker.address.default'] = 'c6401.ambari.apache.org:8083'
+ expected['hive-interactive-site']['properties']['hive.druid.broker.address.default'] = 'c6401.ambari.apache.org:8083'
recommendedConfigurations = {}
self.stackAdvisor.recommendHIVEConfigurations(recommendedConfigurations, clusterData, services, hosts)