You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2016/10/24 20:49:53 UTC
[2/3] ambari git commit: AMBARI-17981. Integrate Druid with Ambari
(Nishant Bangarwa, Slim Bouguerra via Swapan Shridhar).
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
new file mode 100644
index 0000000..ee1d61c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management.libraries.script.script import Script
+
+config = Script.get_config()
+
+druid_pid_dir = config['configurations']['druid-env']['druid_pid_dir']
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
new file mode 100644
index 0000000..c68b9b9
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
@@ -0,0 +1,37 @@
+{
+ "name": "default",
+ "description": "default quick links configuration",
+ "configuration": {
+ "protocol": {
+ "type": "HTTP_ONLY"
+ },
+ "links": [
+ {
+ "name": "coordinator_console",
+ "label": "Druid Coordinator Console",
+ "component_name": "DRUID_COORDINATOR",
+ "requires_user_name": "false",
+ "url": "%@://%@:%@",
+ "port": {
+ "http_property": "druid.port",
+ "http_default_port": "8081",
+ "regex": "^(\\d+)$",
+ "site": "druid-coordinator"
+ }
+ },
+ {
+ "name": "overlord_console",
+ "label": "Druid Overlord Console",
+ "component_name": "DRUID_OVERLORD",
+ "requires_user_name": "false",
+ "url": "%@://%@:%@",
+ "port": {
+ "http_property": "druid.port",
+ "http_default_port": "8090",
+ "regex": "^(\\d+)$",
+ "site": "druid-overlord"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
new file mode 100644
index 0000000..f494594
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
@@ -0,0 +1,120 @@
+{
+ "name": "default",
+ "description": "Default theme for Druid service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "default",
+ "tabs": [
+ {
+ "name": "metadata_storage",
+ "display-name": "META DATA STORAGE CONFIG",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "1",
+ "sections": [
+ {
+ "name": "section-metadata-storage",
+ "display-name": "",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "2",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-metadata-storage-row1-col1",
+ "display-name": "META DATA STORAGE",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "druid-common/database_name",
+ "subsection-name": "subsection-metadata-storage-row1-col1"
+ },
+ {
+ "config": "druid-common/druid.metadata.storage.type",
+ "subsection-name": "subsection-metadata-storage-row1-col1"
+ },
+ {
+ "config": "druid-common/druid.metadata.storage.connector.user",
+ "subsection-name": "subsection-metadata-storage-row1-col1"
+ },
+ {
+ "config": "druid-common/druid.metadata.storage.connector.password",
+ "subsection-name": "subsection-metadata-storage-row1-col1"
+ },
+ {
+ "config": "druid-common/metastore_hostname",
+ "subsection-name": "subsection-metadata-storage-row1-col1"
+ },
+ {
+ "config": "druid-common/druid.metadata.storage.connector.port",
+ "subsection-name": "subsection-metadata-storage-row1-col1"
+ },
+ {
+ "config": "druid-common/druid.metadata.storage.connector.connectURI",
+ "subsection-name": "subsection-metadata-storage-row1-col1"
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "druid-common/database_name",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "druid-common/druid.metadata.storage.type",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
+ "config": "druid-common/druid.metadata.storage.connector.user",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "druid-common/druid.metadata.storage.connector.password",
+ "widget": {
+ "type": "password"
+ }
+ },
+ {
+ "config": "druid-common/metastore_hostname",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "druid-common/druid.metadata.storage.connector.port",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "druid-common/druid.metadata.storage.connector.connectURI",
+ "widget": {
+ "type": "text-field"
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json
new file mode 100644
index 0000000..66b6ed2
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/role_command_order.json
@@ -0,0 +1,19 @@
+{
+ "_comment" : "Record format:",
+ "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2, ...]",
+ "general_deps" : {
+ "_comment" : "dependencies for all cases",
+ "DRUID_HISTORICAL-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_OVERLORD-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_MIDDLEMANAGER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_BROKER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_ROUTER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_COORDINATOR-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "DRUID_OVERLORD-RESTART" : ["DRUID_HISTORICAL-RESTART"],
+ "DRUID_MIDDLEMANAGER-RESTART" : ["DRUID_OVERLORD-RESTART"],
+ "DRUID_BROKER-RESTART" : ["DRUID_MIDDLEMANAGER-RESTART"],
+ "DRUID_ROUTER-RESTART" : ["DRUID_BROKER-RESTART"],
+ "DRUID_COORDINATOR-RESTART" : ["DRUID_ROUTER-RESTART"],
+ "DRUID_SERVICE_CHECK-SERVICE_CHECK" : ["DRUID_HISTORICAL-START", "DRUID_COORDINATOR-START", "DRUID_OVERLORD-START", "DRUID_MIDDLEMANAGER-START", "DRUID_BROKER-START", "DRUID_ROUTER-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json
new file mode 100644
index 0000000..1661285
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/kerberos.json
@@ -0,0 +1,78 @@
+{
+ "services": [
+ {
+ "name": "DRUID",
+ "identities": [
+ {
+ "name": "/spnego"
+ },
+ {
+ "name": "druid",
+ "principal": {
+ "value": "${druid-env/druid_user}@${realm}",
+ "type": "user",
+ "configuration": "druid-common/druid.hadoop.security.kerberos.principal",
+ "local_username": "${druid-env/druid_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/druid.headless.keytab",
+ "owner": {
+ "name": "${druid-env/druid_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": "r"
+ },
+ "configuration": "druid-common/druid.hadoop.security.kerberos.keytab"
+ }
+ },
+ {
+ "name": "/smokeuser"
+ }
+ ],
+ "components": [
+ {
+ "name": "DRUID_HISTORICAL",
+ "identities": [
+ {
+ "name": "/druid"
+ }
+ ]
+ },
+ {
+ "name": "DRUID_BROKER",
+ "identities": [
+ {
+ "name": "/druid"
+ }
+ ]
+ },
+ {
+ "name": "DRUID_OVERLORD",
+ "identities": [
+ {
+ "name": "/druid"
+ }
+ ]
+ },
+ {
+ "name": "DRUID_COORDINATOR",
+ "identities": [
+ {
+ "name": "/druid"
+ }
+ ]
+ },
+ {
+ "name": "DRUID_MIDDLEMANAGER",
+ "identities": [
+ {
+ "name": "/druid"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml
new file mode 100644
index 0000000..49d09b1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>DRUID</name>
+ <version>0.9.2.2.6</version>
+ <extends>common-services/DRUID/0.9.2</extends>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 1f722dc..49dd086 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -16,6 +16,191 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+from resource_management.core.logger import Logger
+import json
+from resource_management.libraries.functions import format
+
class HDP26StackAdvisor(HDP25StackAdvisor):
- pass
\ No newline at end of file
+ def __init__(self):
+ super(HDP26StackAdvisor, self).__init__()
+ Logger.initialize_logger()
+
+ def getServiceConfigurationRecommenderDict(self):
+ parentRecommendConfDict = super(HDP26StackAdvisor, self).getServiceConfigurationRecommenderDict()
+ childRecommendConfDict = {
+ "DRUID": self.recommendDruidConfigurations
+ }
+ parentRecommendConfDict.update(childRecommendConfDict)
+ return parentRecommendConfDict
+
+ def recommendDruidConfigurations(self, configurations, clusterData, services, hosts):
+
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ putCommonProperty = self.putProperty(configurations, "druid-common", services)
+
+ putCommonProperty('druid.zk.service.host', self.getZKHostPortString(services))
+ self.recommendDruidMaxMemoryLimitConfigurations(configurations, clusterData, services, hosts)
+
+ # recommending the metadata storage uri
+ database_name = services['configurations']["druid-common"]["properties"]["database_name"]
+ metastore_hostname = services['configurations']["druid-common"]["properties"]["metastore_hostname"]
+ database_type = services['configurations']["druid-common"]["properties"]["druid.metadata.storage.type"]
+ metadata_storage_port = "1527"
+ mysql_extension_name = "io.druid.extensions:mysql-metadata-storage"
+ mysql_module_name = "mysql-metadata-storage"
+ postgres_module_name = "postgresql-metadata-storage"
+ extensions_load_list = services['configurations']['druid-common']['properties']['druid.extensions.loadList']
+ extensions_pull_list = services['configurations']['druid-common']['properties']['druid.extensions.pullList']
+ putDruidCommonProperty = self.putProperty(configurations, "druid-common", services)
+
+ extensions_pull_list = self.removeFromList(extensions_pull_list, mysql_extension_name)
+ extensions_load_list = self.removeFromList(extensions_load_list, mysql_module_name)
+ extensions_load_list = self.removeFromList(extensions_load_list, postgres_module_name)
+
+ if database_type == 'mysql':
+ metadata_storage_port = "3306"
+ extensions_pull_list = self.addToList(extensions_pull_list, mysql_extension_name)
+ extensions_load_list = self.addToList(extensions_load_list, mysql_module_name)
+
+ if database_type == 'postgres':
+ extensions_load_list = self.addToList(extensions_load_list, postgres_module_name)
+ metadata_storage_port = "5432"
+
+ putDruidCommonProperty('druid.metadata.storage.connector.port', metadata_storage_port)
+ putDruidCommonProperty('druid.metadata.storage.connector.connectURI',
+ self.getMetadataConnectionString(database_type).format(metastore_hostname, database_name,
+ metadata_storage_port))
+ # HDFS is installed
+ if "HDFS" in servicesList and "hdfs-site" in services["configurations"]:
+ # recommend HDFS as default deep storage
+ extensions_load_list = self.addToList(extensions_load_list, "druid-hdfs-storage")
+ putCommonProperty("druid.storage.type", "hdfs")
+ putCommonProperty("druid.storage.storageDirectory", "/user/druid/data")
+ # configure indexer logs configs
+ putCommonProperty("druid.indexer.logs.type", "hdfs")
+ putCommonProperty("druid.indexer.logs.directory", "/user/druid/logs")
+
+ if "KAFKA" in servicesList:
+ extensions_load_list = self.addToList(extensions_load_list, "druid-kafka-indexing-service")
+
+ putCommonProperty('druid.extensions.loadList', extensions_load_list)
+ putCommonProperty('druid.extensions.pullList', extensions_pull_list)
+
+ # JVM Configs go to env properties
+ putEnvProperty = self.putProperty(configurations, "druid-env", services)
+
+ # processing thread pool Config
+ for component in ['DRUID_HISTORICAL', 'DRUID_BROKER']:
+ component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts)
+ nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component]
+ putComponentProperty = self.putProperty(configurations, format("druid-{nodeType}"), services)
+ if (component_hosts is not None and len(component_hosts) > 0):
+ totalAvailableCpu = self.getMinCpu(component_hosts)
+ processingThreads = 1
+ if totalAvailableCpu > 1:
+ processingThreads = totalAvailableCpu - 1
+ putComponentProperty('druid.processing.numThreads', processingThreads)
+ putComponentProperty('druid.server.http.numThreads', max(10, (totalAvailableCpu * 17) / 16 + 2) + 30)
+
+ def getMetadataConnectionString(self, database_type):
+ driverDict = {
+ 'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true',
+ 'derby': 'jdbc:derby://{0}:{2}/{1};create=true',
+ 'postgres': 'jdbc:postgresql://{0}:{2}/{1}'
+ }
+ return driverDict.get(database_type.lower())
+
+ def addToList(self, json_list, word):
+ desr_list = json.loads(json_list)
+ if word not in desr_list:
+ desr_list.append(word)
+ return json.dumps(desr_list)
+
+ def removeFromList(self, json_list, word):
+ desr_list = json.loads(json_list)
+ if word in desr_list:
+ desr_list.remove(word)
+ return json.dumps(desr_list)
+
+ def recommendDruidMaxMemoryLimitConfigurations(self, configurations, clusterData, services, hosts):
+ putEnvPropertyAttribute = self.putPropertyAttribute(configurations, "druid-env")
+ for component in ["DRUID_HISTORICAL", "DRUID_MIDDLEMANAGER", "DRUID_BROKER", "DRUID_OVERLORD",
+ "DRUID_COORDINATOR"]:
+ component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts)
+ if component_hosts is not None and len(component_hosts) > 0:
+ totalAvailableMem = self.getMinMemory(component_hosts) / 1024 # In MB
+ nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component]
+ putEnvPropertyAttribute(format('druid.{nodeType}.jvm.heap.memory'), 'maximum',
+ max(totalAvailableMem, 1024))
+
+ DRUID_COMPONENT_NODE_TYPE_MAP = {
+ 'DRUID_BROKER': 'broker',
+ 'DRUID_COORDINATOR': 'coordinator',
+ 'DRUID_HISTORICAL': 'historical',
+ 'DRUID_MIDDLEMANAGER': 'middlemanager',
+ 'DRUID_OVERLORD': 'overlord',
+ 'DRUID_ROUTER': 'router'
+ }
+
+ def getMinMemory(self, component_hosts):
+ min_ram_kb = 1073741824 # 1 TB
+ for host in component_hosts:
+ ram_kb = host['Hosts']['total_mem']
+ min_ram_kb = min(min_ram_kb, ram_kb)
+ return min_ram_kb
+
+ def getMinCpu(self, component_hosts):
+ min_cpu = 256
+ for host in component_hosts:
+ cpu_count = host['Hosts']['cpu_count']
+ min_cpu = min(min_cpu, cpu_count)
+ return min_cpu
+
+ def getServiceConfigurationValidators(self):
+ parentValidators = super(HDP26StackAdvisor, self).getServiceConfigurationValidators()
+ childValidators = {
+ "DRUID": {"druid-env": self.validateDruidEnvConfigurations,
+ "druid-historical": self.validateDruidHistoricalConfigurations,
+ "druid-broker": self.validateDruidBrokerConfigurations}
+ }
+ self.mergeValidators(parentValidators, childValidators)
+ return parentValidators
+
+ def validateDruidEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = []
+ # Minimum Direct memory Validation
+ envProperties = services['configurations']['druid-env']['properties']
+ for nodeType in ['broker', 'historical']:
+ properties = services['configurations'][format('druid-{nodeType}')]['properties']
+ intermediateBufferSize = int(properties['druid.processing.buffer.sizeBytes']) / (1024 * 1024) # In MBs
+ processingThreads = int(properties['druid.processing.numThreads'])
+ directMemory = int(envProperties[format('druid.{nodeType}.jvm.direct.memory')])
+ if directMemory < (processingThreads + 1) * intermediateBufferSize:
+ validationItems.extend(
+ {"config-name": format("druid.{nodeType}.jvm.direct.memory"), "item": self.getErrorItem(
+ format(
+ "Not enough direct memory available for {nodeType} Node."
+ "Please adjust druid.{nodeType}.jvm.direct.memory, druid.processing.buffer.sizeBytes, druid.processing.numThreads"
+ )
+ )
+ })
+ return self.toConfigurationValidationProblems(validationItems, "druid-env")
+
+ def validateDruidHistoricalConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = [
+ {"config-name": "druid.processing.numThreads",
+ "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults,
+ "druid.processing.numThreads")}
+ ]
+ return self.toConfigurationValidationProblems(validationItems, "druid-historical")
+
+ def validateDruidBrokerConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = [
+ {"config-name": "druid.processing.numThreads",
+ "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults,
+ "druid.processing.numThreads")}
+ ]
+ return self.toConfigurationValidationProblems(validationItems, "druid-broker")
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
new file mode 100644
index 0000000..2988169
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
@@ -0,0 +1,647 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+
+from stacks.utils.RMFTestCase import *
+
+from mock.mock import MagicMock, patch
+from resource_management.libraries import functions
+from resource_management.libraries.functions import format
+from resource_management.core.logger import Logger
+
+@patch("resource_management.libraries.Script.get_tmp_dir", new=MagicMock(return_value=('/var/lib/ambari-agent/tmp')))
+@patch.object(functions, "get_stack_version", new=MagicMock(return_value="2.0.0.0-1234"))
+class TestDruid(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "DRUID/0.9.2/package"
+ STACK_VERSION = "2.6"
+ DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp']
+
+ def setUp(self):
+ Logger.logger = MagicMock()
+ self.testDirectory = os.path.dirname(os.path.abspath(__file__))
+ self.num_times_to_iterate = 3
+ self.wait_time = 1
+
+ def test_configure_overlord(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/overlord.py",
+ classname="DruidOverlord",
+ command="configure",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ config_overrides = { 'role' : 'DRUID_OVERLORD' },
+ stack_version=self.STACK_VERSION,
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-overlord')
+ self.assertNoMoreResources()
+
+ def test_start_overlord(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/overlord.py",
+ classname="DruidOverlord",
+ command="start",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_OVERLORD' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-overlord')
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-overlord/conf/druid-env.sh ; /usr/hdp/current/druid-overlord/bin/node.sh overlord start'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def test_stop_overlord(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/overlord.py",
+ classname="DruidOverlord",
+ command="stop",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_OVERLORD' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-overlord/conf/druid-env.sh ; /usr/hdp/current/druid-overlord/bin/node.sh overlord stop'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def test_configure_coordinator(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/coordinator.py",
+ classname="DruidCoordinator",
+ command="configure",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ config_overrides = { 'role' : 'DRUID_COORDINATOR' },
+ stack_version=self.STACK_VERSION,
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-coordinator')
+ self.assertNoMoreResources()
+
+ def test_start_coordinator(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/coordinator.py",
+ classname="DruidCoordinator",
+ command="start",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_COORDINATOR' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-coordinator')
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-coordinator/conf/druid-env.sh ; /usr/hdp/current/druid-coordinator/bin/node.sh coordinator start'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def test_stop_coordinator(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/coordinator.py",
+ classname="DruidCoordinator",
+ command="stop",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_COORDINATOR' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-coordinator/conf/druid-env.sh ; /usr/hdp/current/druid-coordinator/bin/node.sh coordinator stop'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def test_configure_broker(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/broker.py",
+ classname="DruidBroker",
+ command="configure",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ config_overrides = { 'role' : 'DRUID_BROKER' },
+ stack_version=self.STACK_VERSION,
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-broker')
+ self.assertNoMoreResources()
+
+ def test_start_broker(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/broker.py",
+ classname="DruidBroker",
+ command="start",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_BROKER' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-broker')
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-broker/conf/druid-env.sh ; /usr/hdp/current/druid-broker/bin/node.sh broker start'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def test_stop_broker(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/broker.py",
+ classname="DruidBroker",
+ command="stop",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_BROKER' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-broker/conf/druid-env.sh ; /usr/hdp/current/druid-broker/bin/node.sh broker stop'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def test_configure_router(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/router.py",
+ classname="DruidRouter",
+ command="configure",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ config_overrides = { 'role' : 'DRUID_ROUTER' },
+ stack_version=self.STACK_VERSION,
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-router')
+ self.assertNoMoreResources()
+
+ def test_start_router(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/router.py",
+ classname="DruidRouter",
+ command="start",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_ROUTER' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-router')
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-router/conf/druid-env.sh ; /usr/hdp/current/druid-router/bin/node.sh router start'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def test_stop_router(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/router.py",
+ classname="DruidRouter",
+ command="stop",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_ROUTER' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-router/conf/druid-env.sh ; /usr/hdp/current/druid-router/bin/node.sh router stop'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def test_configure_historical(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historical.py",
+ classname="DruidHistorical",
+ command="configure",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ config_overrides = { 'role' : 'DRUID_HISTORICAL' },
+ stack_version=self.STACK_VERSION,
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-historical')
+ self.assertNoMoreResources()
+
+ def test_start_historical(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historical.py",
+ classname="DruidHistorical",
+ command="start",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_HISTORICAL' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-historical')
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-historical/conf/druid-env.sh ; /usr/hdp/current/druid-historical/bin/node.sh historical start'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def test_stop_historical(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historical.py",
+ classname="DruidHistorical",
+ command="stop",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_HISTORICAL' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-historical/conf/druid-env.sh ; /usr/hdp/current/druid-historical/bin/node.sh historical stop'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def test_configure_middleManager(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/middlemanager.py",
+ classname="DruidMiddleManager",
+ command="configure",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ config_overrides = { 'role' : 'DRUID_MIDDLEMANAGER' },
+ stack_version=self.STACK_VERSION,
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-middlemanager')
+ self.assertNoMoreResources()
+
+ def test_start_middleManager(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/middlemanager.py",
+ classname="DruidMiddleManager",
+ command="start",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_MIDDLEMANAGER' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default('druid-middlemanager')
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-middlemanager/conf/druid-env.sh ; /usr/hdp/current/druid-middlemanager/bin/node.sh middleManager start'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def test_stop_middleManager(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/middlemanager.py",
+ classname="DruidMiddleManager",
+ command="stop",
+ config_file=self.get_src_folder() + "/test/python/stacks/2.6/configs/default.json",
+ stack_version=self.STACK_VERSION,
+ config_overrides = { 'role' : 'DRUID_MIDDLEMANAGER' },
+ target=RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', format('source /usr/hdp/current/druid-middlemanager/conf/druid-env.sh ; /usr/hdp/current/druid-middlemanager/bin/node.sh middleManager stop'),
+ user='druid'
+ )
+ self.assertNoMoreResources()
+
+ def assert_configure_default(self, role):
+
+ self.assertResourceCalled('Directory', '/var/log/druid',
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', '/var/run/druid',
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/_common'),
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/coordinator'),
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/broker'),
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/middleManager'),
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/historical'),
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/overlord'),
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/conf/router'),
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', '/apps/druid/segmentCache',
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/druid-env.sh'),
+ owner = 'druid',
+ content = InlineTemplate(self.getConfig()['configurations']['druid-env']['content'])
+ )
+ druid_common_config = mutable_config_dict(self.getConfig()['configurations']['druid-common'])
+ druid_common_config['druid.host'] = 'c6401.ambari.apache.org'
+ druid_common_config['druid.extensions.directory'] = format('/usr/hdp/current/{role}/extensions')
+ druid_common_config['druid.extensions.hadoopDependenciesDir'] = format('/usr/hdp/current/{role}/hadoop-dependencies')
+ druid_common_config['druid.selectors.indexing.serviceName'] = 'druid/overlord'
+ druid_common_config['druid.selectors.coordinator.serviceName'] = 'druid/coordinator'
+
+ self.assertResourceCalled('PropertiesFile', 'common.runtime.properties',
+ dir=format("/usr/hdp/current/{role}/conf/_common"),
+ properties=druid_common_config,
+ owner='druid',
+ group='hadoop'
+ )
+
+ self.assertResourceCalled('File', format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml'),
+ mode=0644,
+ owner = 'druid',
+ group = 'hadoop',
+ content = InlineTemplate(self.getConfig()['configurations']['druid-log4j']['content'])
+ )
+
+ self.assertResourceCalled('File', '/etc/logrotate.d/druid',
+ mode=0644,
+ owner = 'root',
+ group = 'root',
+ content = InlineTemplate(self.getConfig()['configurations']['druid-logrotate']['content'])
+ )
+
+ self.assertResourceCalled('XmlConfig', "core-site.xml",
+ conf_dir=format('/usr/hdp/current/{role}/conf/_common'),
+ configurations=self.getConfig()['configurations']['core-site'],
+ configuration_attributes=self.getConfig()['configuration_attributes']['core-site'],
+ owner='druid',
+ group='hadoop'
+ )
+
+ self.assertResourceCalled('XmlConfig', "yarn-site.xml",
+ conf_dir=format('/usr/hdp/current/{role}/conf/_common'),
+ configurations=self.getConfig()['configurations']['yarn-site'],
+ configuration_attributes=self.getConfig()['configuration_attributes']['yarn-site'],
+ owner='druid',
+ group='hadoop'
+ )
+
+ self.assertResourceCalled('XmlConfig', "hdfs-site.xml",
+ conf_dir=format('/usr/hdp/current/{role}/conf/_common'),
+ configurations=self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes=self.getConfig()['configuration_attributes']['hdfs-site'],
+ owner='druid',
+ group='hadoop'
+ )
+
+ self.assertResourceCalled('PropertiesFile', "runtime.properties",
+ dir=format('/usr/hdp/current/{role}/conf/coordinator'),
+ properties=self.getConfig()['configurations']['druid-coordinator'],
+ owner='druid',
+ group='hadoop'
+ )
+
+ self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/coordinator/jvm.config"),
+ owner='druid',
+ group='hadoop',
+ content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+ node_heap_memory=1024,
+ node_direct_memory=2048,
+ node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+ log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+ )
+ )
+
+ self.assertResourceCalled('PropertiesFile', "runtime.properties",
+ dir=format('/usr/hdp/current/{role}/conf/overlord'),
+ properties=self.getConfig()['configurations']['druid-overlord'],
+ owner='druid',
+ group='hadoop'
+ )
+
+ self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/overlord/jvm.config"),
+ owner='druid',
+ group='hadoop',
+ content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+ node_heap_memory=1024,
+ node_direct_memory=2048,
+ node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+ log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+ )
+ )
+
+ self.assertResourceCalled('PropertiesFile', "runtime.properties",
+ dir=format('/usr/hdp/current/{role}/conf/historical'),
+ properties=self.getConfig()['configurations']['druid-historical'],
+ owner='druid',
+ group='hadoop'
+ )
+
+ self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/historical/jvm.config"),
+ owner='druid',
+ group='hadoop',
+ content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+ node_heap_memory=1024,
+ node_direct_memory=2048,
+ node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+ log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+ )
+ )
+
+
+ self.assertResourceCalled('PropertiesFile', "runtime.properties",
+ dir=format('/usr/hdp/current/{role}/conf/broker'),
+ properties=self.getConfig()['configurations']['druid-broker'],
+ owner='druid',
+ group='hadoop'
+ )
+
+ self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/broker/jvm.config"),
+ owner='druid',
+ group='hadoop',
+ content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+ node_heap_memory=1024,
+ node_direct_memory=2048,
+ node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+ log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+ )
+ )
+
+
+ self.assertResourceCalled('PropertiesFile', "runtime.properties",
+ dir=format('/usr/hdp/current/{role}/conf/middleManager'),
+ properties=self.getConfig()['configurations']['druid-middlemanager'],
+ owner='druid',
+ group='hadoop'
+ )
+
+ self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/middleManager/jvm.config"),
+ owner='druid',
+ group='hadoop',
+ content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+ node_heap_memory=1024,
+ node_direct_memory=2048,
+ node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+ log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+ )
+ )
+
+ self.assertResourceCalled('PropertiesFile', "runtime.properties",
+ dir=format('/usr/hdp/current/{role}/conf/router'),
+ properties=self.getConfig()['configurations']['druid-router'],
+ owner='druid',
+ group='hadoop'
+ )
+
+ self.assertResourceCalled('File', format("/usr/hdp/current/{role}/conf/router/jvm.config"),
+ owner='druid',
+ group='hadoop',
+ content=InlineTemplate("-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
+ node_heap_memory=1024,
+ node_direct_memory=2048,
+ node_jvm_opts='-Duser.timezone=UTC -Dfile.encoding=UTF-8',
+ log4j_config_file=format('/usr/hdp/current/{role}/conf/_common/druid-log4j.xml')
+ )
+ )
+
+ self.assertResourceCalled('HdfsResource', '/user/druid',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = {u'a': u'b'},
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = 'missing_principal',
+ user = 'hdfs',
+ owner = 'druid',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ type = 'directory',
+ action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ dfs_type = '',
+ recursive_chown=True,
+ recursive_chmod=True
+ )
+
+ self.assertResourceCalled('HdfsResource', '/user/druid/data',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = {u'a': u'b'},
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = 'missing_principal',
+ user = 'hdfs',
+ owner = 'druid',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ type = 'directory',
+ action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ dfs_type = '',
+ mode=0755
+ )
+
+ self.assertResourceCalled('HdfsResource', '/tmp/druid-indexing',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = {u'a': u'b'},
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = 'missing_principal',
+ user = 'hdfs',
+ owner = 'druid',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ type = 'directory',
+ action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ dfs_type = '',
+ mode=0755
+ )
+
+ self.assertResourceCalled('HdfsResource', '/user/druid/logs',
+ immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = {u'a': u'b'},
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = 'missing_principal',
+ user = 'hdfs',
+ owner = 'druid',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ type = 'directory',
+ action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+ dfs_type = '',
+ mode=0755
+ )
+
+ self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/extensions'),
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Directory', format('/usr/hdp/current/{role}/hadoop-dependencies'),
+ mode=0755,
+ cd_access='a',
+ owner='druid',
+ group='hadoop',
+ create_parents=True,
+ recursive_ownership=True
+ )
+
+ self.assertResourceCalled('Execute', format("source /usr/hdp/current/{role}/conf/druid-env.sh ; java -classpath '/usr/hdp/current/{role}/lib/*' -Ddruid.extensions.loadList=[] -Ddruid.extensions.directory=/usr/hdp/current/{role}/extensions -Ddruid.extensions.hadoopDependenciesDir=/usr/hdp/current/{role}/hadoop-dependencies io.druid.cli.Main tools pull-deps -c mysql-metadata-storage --no-default-hadoop"),
+ user='druid'
+ )
+
+
+def mutable_config_dict(config):
+ rv = {}
+ for key, value in config.iteritems():
+ rv[key] = value
+ return rv
http://git-wip-us.apache.org/repos/asf/ambari/blob/dcbd2c07/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
new file mode 100644
index 0000000..8227d69
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
@@ -0,0 +1,575 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import json
+import os
+from unittest import TestCase
+from mock.mock import patch
+
+
+class TestHDP26StackAdvisor(TestCase):
+ def setUp(self):
+ import imp
+ self.maxDiff = None
+ self.testDirectory = os.path.dirname(os.path.abspath(__file__))
+ stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py')
+ hdp206StackAdvisorPath = os.path.join(self.testDirectory,
+ '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py')
+ hdp21StackAdvisorPath = os.path.join(self.testDirectory,
+ '../../../../../main/resources/stacks/HDP/2.1/services/stack_advisor.py')
+ hdp22StackAdvisorPath = os.path.join(self.testDirectory,
+ '../../../../../main/resources/stacks/HDP/2.2/services/stack_advisor.py')
+ hdp23StackAdvisorPath = os.path.join(self.testDirectory,
+ '../../../../../main/resources/stacks/HDP/2.3/services/stack_advisor.py')
+ hdp24StackAdvisorPath = os.path.join(self.testDirectory,
+ '../../../../../main/resources/stacks/HDP/2.4/services/stack_advisor.py')
+ hdp25StackAdvisorPath = os.path.join(self.testDirectory,
+ '../../../../../main/resources/stacks/HDP/2.5/services/stack_advisor.py')
+ hdp26StackAdvisorPath = os.path.join(self.testDirectory,
+ '../../../../../main/resources/stacks/HDP/2.6/services/stack_advisor.py')
+ hdp26StackAdvisorClassName = 'HDP26StackAdvisor'
+
+ with open(stackAdvisorPath, 'rb') as fp:
+ imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ with open(hdp206StackAdvisorPath, 'rb') as fp:
+ imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ with open(hdp21StackAdvisorPath, 'rb') as fp:
+ imp.load_module('stack_advisor_impl', fp, hdp21StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ with open(hdp22StackAdvisorPath, 'rb') as fp:
+ imp.load_module('stack_advisor_impl', fp, hdp22StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ with open(hdp23StackAdvisorPath, 'rb') as fp:
+ imp.load_module('stack_advisor_impl', fp, hdp23StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ with open(hdp24StackAdvisorPath, 'rb') as fp:
+ imp.load_module('stack_advisor_impl', fp, hdp24StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ with open(hdp25StackAdvisorPath, 'rb') as fp:
+ imp.load_module('stack_advisor_impl', fp, hdp25StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+ with open(hdp26StackAdvisorPath, 'rb') as fp:
+ stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp26StackAdvisorPath,
+ ('.py', 'rb', imp.PY_SOURCE))
+ clazz = getattr(stack_advisor_impl, hdp26StackAdvisorClassName)
+ self.stackAdvisor = clazz()
+
+ # substitute method in the instance
+ self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid
+ self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic
+
+ @patch('__builtin__.open')
+ @patch('os.path.exists')
+ def get_system_min_uid_magic(self, exists_mock, open_mock):
+ class MagicFile(object):
+ def read(self):
+ return """
+ #test line UID_MIN 200
+ UID_MIN 500
+ """
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ pass
+
+ def __enter__(self):
+ return self
+
+ exists_mock.return_value = True
+ open_mock.return_value = MagicFile()
+ return self.get_system_min_uid_real()
+
+ def test_recommendDruidConfigurations_withMysql(self):
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "cpu_count": 4,
+ "total_mem": 50331648,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ },
+ }
+ ]
+ }
+
+ services = {
+ "Versions": {
+ "parent_stack_version": "2.5",
+ "stack_name": "HDP",
+ "stack_version": "2.6",
+ "stack_hierarchy": {
+ "stack_name": "HDP",
+ "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"]
+ }
+ },
+ "services": [{
+ "StackServices": {
+ "service_name": "DRUID",
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_COORDINATOR",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_OVERLORD",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_BROKER",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_HISTORICAL",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_MIDDLEMANAGER",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ }
+ ]
+ }
+ ],
+ "configurations": {
+ "druid-common": {
+ "properties": {
+ "database_name": "druid",
+ "metastore_hostname": "c6401.ambari.apache.org",
+ "druid.metadata.storage.type": "mysql",
+ "druid.extensions.loadList": "[\"postgresql-metadata-storage\"]",
+ "druid.extensions.pullList": "[]"
+ }
+ }
+ }
+ }
+
+ clusterData = {
+ "cpu": 4,
+ "mapMemory": 30000,
+ "amMemory": 20000,
+ "reduceMemory": 20560,
+ "containers": 30,
+ "ramPerContainer": 512,
+ "referenceNodeManagerHost": {
+ "total_mem": 10240 * 1024
+ }
+ }
+
+ configurations = {
+ }
+
+ self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations,
+ {'druid-historical': {
+ 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'druid-broker': {
+ 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'druid-common': {'properties': {'druid.extensions.loadList': '["mysql-metadata-storage"]',
+ 'druid.metadata.storage.connector.port': '3306',
+ 'druid.metadata.storage.connector.connectURI': 'jdbc:mysql://c6401.ambari.apache.org:3306/druid?createDatabaseIfNotExist=true',
+ 'druid.zk.service.host': '',
+ 'druid.extensions.pullList': '["io.druid.extensions:mysql-metadata-storage"]'}},
+ 'druid-env': {'properties': {},
+ 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'},
+ 'druid.overlord.jvm.heap.memory': {'maximum': '49152'},
+ 'druid.middlemanager.jvm.heap.memory': {
+ 'maximum': '49152'},
+ 'druid.historical.jvm.heap.memory': {'maximum': '49152'},
+ 'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}}
+ )
+
+ def test_recommendDruidConfigurations_WithPostgresql(self):
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "cpu_count": 4,
+ "total_mem": 50331648,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ }
+ }
+ ]
+ }
+
+ services = {
+ "Versions": {
+ "parent_stack_version": "2.5",
+ "stack_name": "HDP",
+ "stack_version": "2.6",
+ "stack_hierarchy": {
+ "stack_name": "HDP",
+ "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"]
+ }
+ },
+ "services": [{
+ "StackServices": {
+ "service_name": "DRUID",
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_COORDINATOR",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_OVERLORD",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_BROKER",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_HISTORICAL",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_MIDDLEMANAGER",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ }
+ ]
+ }
+ ],
+ "configurations": {
+ "druid-common": {
+ "properties": {
+ "database_name": "druid",
+ "metastore_hostname": "c6401.ambari.apache.org",
+ "druid.metadata.storage.type": "postgres",
+ "druid.extensions.loadList": "[\"mysql-metadata-storage\"]",
+ "druid.extensions.pullList": "[]"
+ }
+ }
+ }
+ }
+
+ clusterData = {
+ }
+
+ configurations = {
+ }
+
+ self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations,
+ {'druid-historical': {
+ 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'druid-broker': {
+ 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'druid-common': {'properties': {'druid.extensions.loadList': '["postgresql-metadata-storage"]',
+ 'druid.metadata.storage.connector.port': '5432',
+ 'druid.metadata.storage.connector.connectURI': 'jdbc:postgresql://c6401.ambari.apache.org:5432/druid',
+ 'druid.zk.service.host': '',
+ 'druid.extensions.pullList': '[]'}},
+ 'druid-env': {'properties': {},
+ 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'},
+ 'druid.overlord.jvm.heap.memory': {'maximum': '49152'},
+ 'druid.middlemanager.jvm.heap.memory': {
+ 'maximum': '49152'},
+ 'druid.historical.jvm.heap.memory': {'maximum': '49152'},
+ 'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}}
+ )
+
+ def test_recommendDruidConfigurations_WithDerby(self):
+ hosts = {
+ "items": [
+ {
+ "Hosts": {
+ "cpu_count": 4,
+ "total_mem": 50331648,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ }
+ }
+ ]
+ }
+
+ services = {
+ "Versions": {
+ "parent_stack_version": "2.5",
+ "stack_name": "HDP",
+ "stack_version": "2.6",
+ "stack_hierarchy": {
+ "stack_name": "HDP",
+ "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"]
+ }
+ },
+ "services": [{
+ "StackServices": {
+ "service_name": "DRUID",
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_COORDINATOR",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_OVERLORD",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_BROKER",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_HISTORICAL",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_MIDDLEMANAGER",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ }
+ ]
+ }
+ ],
+ "configurations": {
+ "druid-common": {
+ "properties": {
+ "database_name": "druid",
+ "metastore_hostname": "c6401.ambari.apache.org",
+ "druid.metadata.storage.type": "derby",
+ "druid.extensions.loadList": "[\"mysql-metadata-storage\"]",
+ "druid.extensions.pullList": "[]"
+ }
+ }
+ }
+ }
+
+ clusterData = {
+ }
+
+ configurations = {
+ }
+
+ self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations,
+ {'druid-historical': {
+ 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'druid-broker': {
+ 'properties': {'druid.processing.numThreads': '3', 'druid.server.http.numThreads': '40'}},
+ 'druid-common': {'properties': {'druid.extensions.loadList': '[]',
+ 'druid.metadata.storage.connector.port': '1527',
+ 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true',
+ 'druid.zk.service.host': '',
+ 'druid.extensions.pullList': '[]'}},
+ 'druid-env': {'properties': {},
+ 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'},
+ 'druid.overlord.jvm.heap.memory': {'maximum': '49152'},
+ 'druid.middlemanager.jvm.heap.memory': {
+ 'maximum': '49152'},
+ 'druid.historical.jvm.heap.memory': {'maximum': '49152'},
+ 'druid.broker.jvm.heap.memory': {'maximum': '49152'}}}}
+ )
+
+ def test_recommendDruidConfigurations_heterogeneous_hosts(self):
+ hosts = {
+ "items": [
+ {
+ "href": "/api/v1/hosts/c6401.ambari.apache.org",
+ "Hosts": {
+ "cpu_count": 4,
+ "total_mem": 50331648,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6401.ambari.apache.org",
+ "host_name": "c6401.ambari.apache.org"
+ }
+ }, {
+ "href": "/api/v1/hosts/c6402.ambari.apache.org",
+ "Hosts": {
+ "cpu_count": 1,
+ "total_mem": 1922680,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6402.ambari.apache.org",
+ "host_name": "c6402.ambari.apache.org"
+ }
+ },
+ {
+ "href": "/api/v1/hosts/c6403.ambari.apache.org",
+ "Hosts": {
+ "cpu_count": 3,
+ "total_mem": 3845360,
+ "disk_info": [
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"},
+ {"mountpoint": "/"},
+ {"mountpoint": "/dev/shm"},
+ {"mountpoint": "/vagrant"}
+ ],
+ "public_host_name": "c6403.ambari.apache.org",
+ "host_name": "c6403.ambari.apache.org"
+ }
+ }
+ ]
+ }
+
+ services = {
+ "Versions": {
+ "parent_stack_version": "2.5",
+ "stack_name": "HDP",
+ "stack_version": "2.6",
+ "stack_hierarchy": {
+ "stack_name": "HDP",
+ "stack_versions": ["2.5", "2.4", "2.3", "2.2", "2.1", "2.0.6"]
+ }
+ },
+ "services": [{
+ "StackServices": {
+ "service_name": "DRUID",
+ },
+ "components": [
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_COORDINATOR",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_OVERLORD",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_BROKER",
+ "hostnames": ["c6402.ambari.apache.org", "c6403.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_HISTORICAL",
+ "hostnames": ["c6401.ambari.apache.org", "c6403.ambari.apache.org"]
+ },
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "DRUID_MIDDLEMANAGER",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ }
+ ]
+ }
+ ],
+ "configurations": {
+ "druid-common": {
+ "properties": {
+ "database_name": "druid",
+ "metastore_hostname": "c6401.ambari.apache.org",
+ "druid.metadata.storage.type": "derby",
+ "druid.extensions.loadList": "[\"mysql-metadata-storage\"]",
+ "druid.extensions.pullList": "[]"
+ }
+ }
+ }
+ }
+
+ clusterData = {
+ }
+
+ configurations = {
+ }
+
+ self.stackAdvisor.recommendDruidConfigurations(configurations, clusterData, services, hosts)
+ self.assertEquals(configurations,
+ {'druid-historical': {
+ 'properties': {'druid.processing.numThreads': '2', 'druid.server.http.numThreads': '40'}},
+ 'druid-broker': {
+ 'properties': {'druid.processing.numThreads': '1', 'druid.server.http.numThreads': '40'}},
+ 'druid-common': {'properties': {'druid.extensions.loadList': '[]',
+ 'druid.metadata.storage.connector.port': '1527',
+ 'druid.metadata.storage.connector.connectURI': 'jdbc:derby://c6401.ambari.apache.org:1527/druid;create=true',
+ 'druid.zk.service.host': '',
+ 'druid.extensions.pullList': '[]'}},
+ 'druid-env': {'properties': {},
+ 'property_attributes': {'druid.coordinator.jvm.heap.memory': {'maximum': '49152'},
+ 'druid.overlord.jvm.heap.memory': {'maximum': '49152'},
+ 'druid.middlemanager.jvm.heap.memory': {
+ 'maximum': '49152'},
+ 'druid.historical.jvm.heap.memory': {'maximum': '3755'},
+ 'druid.broker.jvm.heap.memory': {'maximum': '1877'}}}}
+ )
+
+
+def load_json(self, filename):
+ file = os.path.join(self.testDirectory, filename)
+ with open(file, 'rb') as f:
+ data = json.load(f)
+ return data