You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2017/10/09 06:01:45 UTC
[47/50] [abbrv] ambari git commit: AMBARI-21776. Move druid version
to druid 0.10.1 and drop TP flag. (Slim Bouguerra via Swapan Shridhar).
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py
deleted file mode 100644
index ec98c3c..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid.py
+++ /dev/null
@@ -1,307 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-import json
-import os
-from resource_management import Fail
-from resource_management.libraries.resources.properties_file import PropertiesFile
-from resource_management.core.resources.system import Directory, Execute, File
-from resource_management.core.source import DownloadSource
-from resource_management.core.source import InlineTemplate
-from resource_management.libraries.functions import format
-from resource_management.libraries.functions.show_logs import show_logs
-from resource_management.core.logger import Logger
-
-
-def druid(upgrade_type=None, nodeType=None):
- import params
- ensure_base_directories()
-
- # Environment Variables
- File(format("{params.druid_conf_dir}/druid-env.sh"),
- owner=params.druid_user,
- content=InlineTemplate(params.druid_env_sh_template),
- mode = 0700
- )
-
- # common config
- druid_common_config = mutable_config_dict(params.config['configurations']['druid-common'])
- # User cannot override below configs
- druid_common_config['druid.host'] = params.hostname
- druid_common_config['druid.extensions.directory'] = params.druid_extensions_dir
- druid_common_config['druid.extensions.hadoopDependenciesDir'] = params.druid_hadoop_dependencies_dir
- druid_common_config['druid.selectors.indexing.serviceName'] = params.config['configurations']['druid-overlord'][
- 'druid.service']
- druid_common_config['druid.selectors.coordinator.serviceName'] = \
- params.config['configurations']['druid-coordinator']['druid.service']
- druid_common_config['druid.extensions.loadList'] = json.dumps(eval(params.druid_extensions_load_list) +
- eval(params.druid_security_extensions_load_list))
-
- # delete the password and user if empty otherwiswe derby will fail.
- if 'derby' == druid_common_config['druid.metadata.storage.type']:
- del druid_common_config['druid.metadata.storage.connector.user']
- del druid_common_config['druid.metadata.storage.connector.password']
-
- druid_env_config = mutable_config_dict(params.config['configurations']['druid-env'])
-
- PropertiesFile("common.runtime.properties",
- dir=params.druid_common_conf_dir,
- properties=druid_common_config,
- owner=params.druid_user,
- group=params.user_group,
- mode = 0600
- )
- Logger.info("Created common.runtime.properties")
-
- File(format("{params.druid_common_conf_dir}/druid-log4j.xml"),
- mode=0644,
- owner=params.druid_user,
- group=params.user_group,
- content=InlineTemplate(params.log4j_props)
- )
- Logger.info("Created log4j file")
-
- File("/etc/logrotate.d/druid",
- mode=0644,
- owner='root',
- group='root',
- content=InlineTemplate(params.logrotate_props)
- )
-
- Logger.info("Created log rotate file")
-
- # node specific configs
- for node_type in ['coordinator', 'overlord', 'historical', 'broker', 'middleManager', 'router']:
- node_config_dir = format('{params.druid_conf_dir}/{node_type}')
- node_type_lowercase = node_type.lower()
-
- # Write runtime.properties file
- node_config = mutable_config_dict(params.config['configurations'][format('druid-{node_type_lowercase}')])
- PropertiesFile("runtime.properties",
- dir=node_config_dir,
- properties=node_config,
- owner=params.druid_user,
- group=params.user_group,
- mode = 0600
- )
- Logger.info(format("Created druid-{node_type_lowercase} runtime.properties"))
-
- # Write jvm configs
- File(format('{node_config_dir}/jvm.config'),
- owner=params.druid_user,
- group=params.user_group,
- content=InlineTemplate(
- "-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}",
- node_heap_memory=druid_env_config[format('druid.{node_type_lowercase}.jvm.heap.memory')],
- log4j_config_file=format("{params.druid_common_conf_dir}/druid-log4j.xml"),
- node_direct_memory=druid_env_config[
- format('druid.{node_type_lowercase}.jvm.direct.memory')],
- node_jvm_opts=druid_env_config[format('druid.{node_type_lowercase}.jvm.opts')])
- )
- Logger.info(format("Created druid-{node_type_lowercase} jvm.config"))
- # Handling hadoop Lzo jars if enable and node type is hadoop related eg Overlords and MMs
- if ['middleManager', 'overlord'].__contains__(node_type_lowercase) and params.lzo_enabled and len(
- params.lzo_packages) > 0:
- try:
- Logger.info(
- format(
- "Copying hadoop lzo jars from {hadoop_lib_home} to {druid_hadoop_dependencies_dir}/hadoop-client/*/"))
- Execute(
- format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {druid_hadoop_dependencies_dir}/hadoop-client/*/'))
- except Fail as ex:
- Logger.info(format("No Hadoop LZO found at {hadoop_lib_home}/hadoop-lzo*.jar"))
-
- # All druid nodes have dependency on hdfs_client
- ensure_hadoop_directories()
- download_database_connector_if_needed()
- # Pull all required dependencies
- pulldeps()
-
-
-def mutable_config_dict(config):
- rv = {}
- for key, value in config.iteritems():
- rv[key] = value
- return rv
-
-
-def ensure_hadoop_directories():
- import params
- if 'hdfs-site' not in params.config['configurations']:
- # HDFS Not Installed nothing to do.
- Logger.info("Skipping HDFS directory creation as HDFS not installed")
- return
-
- druid_common_config = params.config['configurations']['druid-common']
- # final overlord config contains both common and overlord config
- druid_middlemanager_config = params.config['configurations']['druid-middlemanager']
-
- # If user is using HDFS as deep storage create HDFS Directory for storing segments
- deep_storage = druid_common_config["druid.storage.type"]
- storage_dir = druid_common_config["druid.storage.storageDirectory"]
-
- if deep_storage == 'hdfs':
- # create the home dir for druid
- params.HdfsResource(format("/user/{params.druid_user}"),
- type="directory",
- action="create_on_execute",
- owner=params.druid_user,
- group='hadoop',
- recursive_chown=True,
- recursive_chmod=True
- )
-
- # create the segment storage dir, users like hive from group hadoop need to write to this directory
- create_hadoop_directory(storage_dir, mode=0775)
-
- # Create HadoopIndexTask hadoopWorkingPath
- hadoop_working_path = druid_middlemanager_config['druid.indexer.task.hadoopWorkingPath']
- if hadoop_working_path is not None:
- if hadoop_working_path.startswith(params.hdfs_tmp_dir):
- params.HdfsResource(params.hdfs_tmp_dir,
- type="directory",
- action="create_on_execute",
- owner=params.hdfs_user,
- mode=0777,
- )
- create_hadoop_directory(hadoop_working_path, mode=0775)
-
- # If HDFS is used for storing logs, create Index Task log directory
- indexer_logs_type = druid_common_config['druid.indexer.logs.type']
- indexer_logs_directory = druid_common_config['druid.indexer.logs.directory']
- if indexer_logs_type == 'hdfs' and indexer_logs_directory is not None:
- create_hadoop_directory(indexer_logs_directory)
-
-
-def create_hadoop_directory(hadoop_dir, mode=0755):
- import params
- params.HdfsResource(hadoop_dir,
- type="directory",
- action="create_on_execute",
- owner=params.druid_user,
- group='hadoop',
- mode=mode
- )
- Logger.info(format("Created Hadoop Directory [{hadoop_dir}], with mode [{mode}]"))
-
-
-def ensure_base_directories():
- import params
- Directory(
- [params.druid_log_dir, params.druid_pid_dir],
- mode=0755,
- owner=params.druid_user,
- group=params.user_group,
- create_parents=True,
- recursive_ownership=True,
- )
-
- Directory(
- [params.druid_conf_dir, params.druid_common_conf_dir, params.druid_coordinator_conf_dir,
- params.druid_broker_conf_dir, params.druid_middlemanager_conf_dir, params.druid_historical_conf_dir,
- params.druid_overlord_conf_dir, params.druid_router_conf_dir, params.druid_segment_infoDir,
- params.druid_tasks_dir],
- mode=0700,
- cd_access='a',
- owner=params.druid_user,
- group=params.user_group,
- create_parents=True,
- recursive_ownership=True,
- )
-
- segment_cache_locations = json.loads(params.druid_segment_cache_locations)
- for segment_cache_location in segment_cache_locations:
- Directory(
- segment_cache_location["path"],
- mode=0700,
- owner=params.druid_user,
- group=params.user_group,
- create_parents=True,
- recursive_ownership=True,
- cd_access='a'
- )
-
-
-
-def get_daemon_cmd(params=None, node_type=None, command=None):
- return format('source {params.druid_conf_dir}/druid-env.sh ; {params.druid_home}/bin/node.sh {node_type} {command}')
-
-
-def getPid(params=None, nodeType=None):
- return format('{params.druid_pid_dir}/{nodeType}.pid')
-
-
-def pulldeps():
- import params
- extensions_list = eval(params.druid_extensions)
- extensions_string = '{0}'.format("-c ".join(extensions_list))
- repository_list = eval(params.druid_repo_list)
- repository_string = '{0}'.format("-r ".join(repository_list))
- if len(extensions_list) > 0:
- try:
- # Make sure druid user has permissions to write dependencies
- Directory(
- [params.druid_extensions_dir, params.druid_hadoop_dependencies_dir],
- mode=0755,
- cd_access='a',
- owner=params.druid_user,
- group=params.user_group,
- create_parents=True,
- recursive_ownership=True,
- )
- pull_deps_command = format(
- "source {params.druid_conf_dir}/druid-env.sh ; java -classpath '{params.druid_home}/lib/*' -Ddruid.extensions.loadList=[] "
- "-Ddruid.extensions.directory={params.druid_extensions_dir} -Ddruid.extensions.hadoopDependenciesDir={params.druid_hadoop_dependencies_dir} "
- "io.druid.cli.Main tools pull-deps -c {extensions_string} --no-default-hadoop")
-
- if len(repository_list) > 0:
- pull_deps_command = format("{pull_deps_command} -r {repository_string}")
-
- Execute(pull_deps_command,
- user=params.druid_user
- )
- Logger.info(format("Pull Dependencies Complete"))
- except:
- show_logs(params.druid_log_dir, params.druid_user)
- raise
-
-
-def download_database_connector_if_needed():
- """
- Downloads the database connector to use when connecting to the metadata storage
- """
- import params
- if params.metadata_storage_type != 'mysql' or not params.jdbc_driver_jar:
- return
-
- File(params.check_db_connection_jar,
- content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}"))
- )
-
- target_jar_with_directory = params.connector_download_dir + os.path.sep + params.jdbc_driver_jar
-
- if not os.path.exists(target_jar_with_directory):
- File(params.downloaded_custom_connector,
- content=DownloadSource(params.connector_curl_source))
-
- Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target_jar_with_directory),
- path=["/bin", "/usr/bin/"],
- sudo=True)
-
- File(target_jar_with_directory, owner=params.druid_user,
- group=params.user_group)
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py
deleted file mode 100644
index 8053dcb..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/druid_node.py
+++ /dev/null
@@ -1,114 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-import os
-from resource_management.core import sudo
-from resource_management import Script
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.format import format
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions.check_process_status import check_process_status
-from resource_management.libraries.functions.show_logs import show_logs
-from druid import druid, get_daemon_cmd, getPid
-
-
-class DruidBase(Script):
- def __init__(self, nodeType=None):
- self.nodeType = nodeType
-
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- druid(upgrade_type=upgrade_type, nodeType=self.nodeType)
-
- def pre_upgrade_restart(self, env, upgrade_type=None):
- node_type_lower = self.nodeType.lower()
- Logger.info(format("Executing druid-{node_type_lower} Upgrade pre-restart"))
- import params
-
- env.set_params(params)
-
- if params.stack_version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version):
- stack_select.select_packages(params.stack_version)
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- self.configure(env, upgrade_type=upgrade_type)
- daemon_cmd = get_daemon_cmd(params, self.nodeType, "start")
- # Verify Database connection on Druid start
- if params.metadata_storage_type == 'mysql':
- if not params.jdbc_driver_jar or not os.path.isfile(params.connector_download_dir + os.path.sep + params.jdbc_driver_jar):
- path_to_jdbc = params.connector_download_dir + os.path.sep + "*"
- error_message = "Error! Sorry, but we can't find jdbc driver for mysql.So, db connection check can fail." + \
- "Please run 'ambari-server setup --jdbc-db=mysql --jdbc-driver={path_to_jdbc} on server host.'"
- Logger.error(error_message)
- else:
- path_to_jdbc = params.connector_download_dir + os.path.sep + params.jdbc_driver_jar
- db_connection_check_command = format("{params.java8_home}/bin/java -cp {params.check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{params.metadata_storage_url}' {params.metadata_storage_user} {params.metadata_storage_password!p} com.mysql.jdbc.Driver")
- else:
- db_connection_check_command = None
-
- if db_connection_check_command:
- sudo.chmod(params.check_db_connection_jar, 0755)
- Execute( db_connection_check_command,
- tries=5,
- try_sleep=10,
- user=params.druid_user
- )
-
- try:
- Execute(daemon_cmd,
- user=params.druid_user
- )
- except:
- show_logs(params.druid_log_dir, params.druid_user)
- raise
-
- def stop(self, env, upgrade_type=None):
- import params
- env.set_params(params)
-
- daemon_cmd = get_daemon_cmd(params, self.nodeType, "stop")
- try:
- Execute(daemon_cmd,
- user=params.druid_user
- )
- except:
- show_logs(params.druid_log_dir, params.druid_user)
- raise
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- pid_file = getPid(status_params, self.nodeType)
- check_process_status(pid_file)
-
- def get_log_folder(self):
- import params
- return params.druid_log_dir
-
- def get_user(self):
- import params
- return params.druid_user
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py
deleted file mode 100644
index 22390a6..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/historical.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from druid_node import DruidBase
-
-
-class DruidHistorical(DruidBase):
- def __init__(self):
- DruidBase.__init__(self, nodeType="historical")
-
-
-if __name__ == "__main__":
- DruidHistorical().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py
deleted file mode 100644
index 20df89c..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/middlemanager.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from druid_node import DruidBase
-
-
-class DruidMiddleManager(DruidBase):
- def __init__(self):
- DruidBase.__init__(self, nodeType="middleManager")
-
-
-if __name__ == "__main__":
- DruidMiddleManager().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py
deleted file mode 100644
index e4d7fcc..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/overlord.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from druid_node import DruidBase
-
-
-class DruidOverlord(DruidBase):
- def __init__(self):
- DruidBase.__init__(self, nodeType="overlord")
-
-
-if __name__ == "__main__":
- DruidOverlord().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
deleted file mode 100644
index fd1cde6..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/params.py
+++ /dev/null
@@ -1,200 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from ambari_commons import OSCheck
-from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
-from resource_management.libraries.functions import conf_select
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
-from resource_management.libraries.functions import get_kinit_path
-from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions import format
-from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
-from resource_management.libraries.functions.default import default
-from ambari_commons.constants import AMBARI_SUDO_BINARY
-
-import status_params
-
-# a map of the Ambari role to the component name
-# for use with <stack-root>/current/<component>
-SERVER_ROLE_DIRECTORY_MAP = {
- 'DRUID_BROKER': 'druid-broker',
- 'DRUID_COORDINATOR': 'druid-coordinator',
- 'DRUID_HISTORICAL': 'druid-historical',
- 'DRUID_MIDDLEMANAGER': 'druid-middlemanager',
- 'DRUID_OVERLORD': 'druid-overlord',
- 'DRUID_ROUTER': 'druid-router'
-}
-
-# server configurations
-config = Script.get_config()
-stack_root = Script.get_stack_root()
-tmp_dir = Script.get_tmp_dir()
-
-stack_name = default("/hostLevelParams/stack_name", None)
-
-# stack version
-stack_version = default("/commandParams/version", None)
-
-# un-formatted stack version
-stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
-
-# default role to coordinator needed for service checks
-component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "DRUID_COORDINATOR")
-
-hostname = config['hostname']
-sudo = AMBARI_SUDO_BINARY
-
-# default druid parameters
-druid_home = format("{stack_root}/current/{component_directory}")
-druid_conf_dir = format("{stack_root}/current/{component_directory}/conf")
-
-druid_common_conf_dir = druid_conf_dir + "/_common"
-druid_coordinator_conf_dir = druid_conf_dir + "/coordinator"
-druid_overlord_conf_dir = druid_conf_dir + "/overlord"
-druid_broker_conf_dir = druid_conf_dir + "/broker"
-druid_historical_conf_dir = druid_conf_dir + "/historical"
-druid_middlemanager_conf_dir = druid_conf_dir + "/middleManager"
-druid_router_conf_dir = druid_conf_dir + "/router"
-druid_extensions_dir = druid_home + "/extensions"
-druid_hadoop_dependencies_dir = druid_home + "/hadoop-dependencies"
-druid_segment_infoDir = config['configurations']['druid-historical']['druid.segmentCache.infoDir']
-druid_segment_cache_locations = config['configurations']['druid-historical']['druid.segmentCache.locations']
-druid_tasks_dir = config['configurations']['druid-middlemanager']['druid.indexer.task.baseTaskDir']
-druid_user = config['configurations']['druid-env']['druid_user']
-druid_log_dir = config['configurations']['druid-env']['druid_log_dir']
-druid_classpath = config['configurations']['druid-env']['druid_classpath']
-druid_extensions = config['configurations']['druid-common']['druid.extensions.pullList']
-druid_repo_list = config['configurations']['druid-common']['druid.extensions.repositoryList']
-druid_extensions_load_list = config['configurations']['druid-common']['druid.extensions.loadList']
-druid_security_extensions_load_list = config['configurations']['druid-common']['druid.security.extensions.loadList']
-
-
-# status params
-druid_pid_dir = status_params.druid_pid_dir
-user_group = config['configurations']['cluster-env']['user_group']
-java8_home = config['hostLevelParams']['java_home']
-druid_env_sh_template = config['configurations']['druid-env']['content']
-
-# log4j params
-log4j_props = config['configurations']['druid-log4j']['content']
-druid_log_level = config['configurations']['druid-log4j']['druid_log_level']
-metamx_log_level = config['configurations']['druid-log4j']['metamx_log_level']
-root_log_level = config['configurations']['druid-log4j']['root_log_level']
-
-druid_log_maxbackupindex = default('/configurations/druid-logrotate/druid_log_maxbackupindex', 7)
-druid_log_maxfilesize = default('/configurations/druid-logrotate/druid_log_maxfilesize', 256)
-logrotate_props = config['configurations']['druid-logrotate']['content']
-
-# Metadata storage
-metadata_storage_user = config['configurations']['druid-common']['druid.metadata.storage.connector.user']
-metadata_storage_password = config['configurations']['druid-common']['druid.metadata.storage.connector.password']
-metadata_storage_db_name = config['configurations']['druid-common']['database_name']
-metadata_storage_db_name = config['configurations']['druid-common']['database_name']
-metadata_storage_type = config['configurations']['druid-common']['druid.metadata.storage.type']
-metadata_storage_url = config['configurations']['druid-common']['druid.metadata.storage.connector.connectURI']
-jdk_location = config['hostLevelParams']['jdk_location']
-if 'mysql' == metadata_storage_type:
- jdbc_driver_jar = default("/hostLevelParams/custom_mysql_jdbc_name", None)
- connector_curl_source = format("{jdk_location}/{jdbc_driver_jar}")
- connector_download_dir=format("{druid_extensions_dir}/mysql-metadata-storage")
- downloaded_custom_connector = format("{tmp_dir}/{jdbc_driver_jar}")
-
-check_db_connection_jar_name = "DBConnectionVerification.jar"
-check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}")
-
-# HDFS
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
-hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
-hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', 'missing_principal').replace("_HOST",
- hostname)
-hdfs_site = config['configurations']['hdfs-site']
-default_fs = config['configurations']['core-site']['fs.defaultFS']
-dfs_type = default("/commandParams/dfs_type", "")
-hdfs_tmp_dir = config['configurations']['hadoop-env']['hdfs_tmp_dir']
-
-# Kerberos
-druid_principal_name = default('/configurations/druid-common/druid.hadoop.security.kerberos.principal',
- 'missing_principal')
-druid_user_keytab = default('/configurations/druid-common/druid.hadoop.security.kerberos.keytab', 'missing_keytab')
-
-import functools
-
-# create partial functions with common arguments for every HdfsResource call
-# to create hdfs directory we need to call params.HdfsResource in code
-HdfsResource = functools.partial(
- HdfsResource,
- user=hdfs_user,
- hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
- security_enabled=security_enabled,
- keytab=hdfs_user_keytab,
- kinit_path_local=kinit_path_local,
- hadoop_bin_dir=hadoop_bin_dir,
- hadoop_conf_dir=hadoop_conf_dir,
- principal_name=hdfs_principal_name,
- hdfs_site=hdfs_site,
- default_fs=default_fs,
- immutable_paths=get_not_managed_resources(),
- dfs_type=dfs_type
-)
-
-# Ambari Metrics
-metric_emitter_type = "noop"
-metric_collector_host = ""
-metric_collector_port = ""
-metric_collector_protocol = ""
-metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
-metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
-metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
-
-ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
-has_metric_collector = not len(ams_collector_hosts) == 0
-
-if has_metric_collector:
- metric_emitter_type = "ambari-metrics"
- if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_host' in config['configurations']['cluster-env']:
- metric_collector_host = config['configurations']['cluster-env']['metrics_collector_vip_host']
- else:
- metric_collector_host = ams_collector_hosts[0]
- if 'cluster-env' in config['configurations'] and \
- 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
- metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
- else:
- metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "localhost:6188")
- if metric_collector_web_address.find(':') != -1:
- metric_collector_port = metric_collector_web_address.split(':')[1]
- else:
- metric_collector_port = '6188'
- if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
- metric_collector_protocol = 'https'
- else:
- metric_collector_protocol = 'http'
- pass
-
-# Create current Hadoop Clients Libs
-stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
-io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-lzo_packages = get_lzo_packages(stack_version_unformatted)
-hadoop_lib_home = stack_root + '/' + stack_version + '/hadoop/lib'
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py
deleted file mode 100644
index 1731a2a..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/router.py
+++ /dev/null
@@ -1,28 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from druid_node import DruidBase
-
-
-class DruidRouter(DruidBase):
- def __init__(self):
- DruidBase.__init__(self, nodeType="router")
-
-
-if __name__ == "__main__":
- DruidRouter().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py
deleted file mode 100644
index 139b727..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/service_check.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions.format import format
-from resource_management.core.resources.system import Execute
-
-
-class ServiceCheck(Script):
- def service_check(self, env):
- import params
- env.set_params(params)
-
- self.checkComponent(params, "druid_coordinator", "druid-coordinator")
- self.checkComponent(params, "druid_overlord", "druid-overlord")
-
- def checkComponent(self, params, component_name, config_name):
- component_port = params.config['configurations'][format('{config_name}')]['druid.port']
- for component_host in params.config['clusterHostInfo'][format('{component_name}_hosts')]:
- Execute(format(
- "curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k {component_host}:{component_port}/status | grep 200"),
- tries=10,
- try_sleep=3,
- logoutput=True)
-
-
-if __name__ == "__main__":
- ServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
deleted file mode 100644
index ee1d61c..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/package/scripts/status_params.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-from resource_management.libraries.script.script import Script
-
-config = Script.get_config()
-
-druid_pid_dir = config['configurations']['druid-env']['druid_pid_dir']
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
deleted file mode 100644
index c68b9b9..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
- "name": "default",
- "description": "default quick links configuration",
- "configuration": {
- "protocol": {
- "type": "HTTP_ONLY"
- },
- "links": [
- {
- "name": "coordinator_console",
- "label": "Druid Coordinator Console",
- "component_name": "DRUID_COORDINATOR",
- "requires_user_name": "false",
- "url": "%@://%@:%@",
- "port": {
- "http_property": "druid.port",
- "http_default_port": "8081",
- "regex": "^(\\d+)$",
- "site": "druid-coordinator"
- }
- },
- {
- "name": "overlord_console",
- "label": "Druid Overlord Console",
- "component_name": "DRUID_OVERLORD",
- "requires_user_name": "false",
- "url": "%@://%@:%@",
- "port": {
- "http_property": "druid.port",
- "http_default_port": "8090",
- "regex": "^(\\d+)$",
- "site": "druid-overlord"
- }
- }
- ]
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json
deleted file mode 100644
index 4d697fe..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/role_command_order.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "general_deps" : {
- "_comment" : "dependencies for Druid",
- "DRUID_HISTORICAL-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_OVERLORD-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_MIDDLEMANAGER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_BROKER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_ROUTER-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_COORDINATOR-START" : ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START", "RESOURCEMANAGER-START", "NODEMANAGER-START"],
- "DRUID_OVERLORD-RESTART" : ["DRUID_HISTORICAL-RESTART"],
- "DRUID_MIDDLEMANAGER-RESTART" : ["DRUID_OVERLORD-RESTART"],
- "DRUID_BROKER-RESTART" : ["DRUID_MIDDLEMANAGER-RESTART"],
- "DRUID_ROUTER-RESTART" : ["DRUID_BROKER-RESTART"],
- "DRUID_COORDINATOR-RESTART" : ["DRUID_ROUTER-RESTART"],
- "DRUID_SERVICE_CHECK-SERVICE_CHECK" : ["DRUID_HISTORICAL-START", "DRUID_COORDINATOR-START", "DRUID_OVERLORD-START", "DRUID_MIDDLEMANAGER-START", "DRUID_BROKER-START", "DRUID_ROUTER-START"]
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json b/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
deleted file mode 100644
index 7033e19..0000000
--- a/ambari-server/src/main/resources/common-services/DRUID/0.9.2/themes/theme.json
+++ /dev/null
@@ -1,120 +0,0 @@
-{
- "name": "default",
- "description": "Default theme for Druid service",
- "configuration": {
- "layouts": [
- {
- "name": "default",
- "tabs": [
- {
- "name": "metadata_storage",
- "display-name": "META DATA STORAGE CONFIG",
- "layout": {
- "tab-columns": "",
- "tab-rows": "1",
- "sections": [
- {
- "name": "section-metadata-storage",
- "display-name": "",
- "row-index": "0",
- "column-index": "0",
- "row-span": "2",
- "column-span": "1",
- "section-columns": "1",
- "section-rows": "1",
- "subsections": [
- {
- "name": "subsection-metadata-storage-row1-col1",
- "display-name": "DRUID META DATA STORAGE",
- "row-index": "0",
- "column-index": "0",
- "row-span": "1",
- "column-span": "1"
- }
- ]
- }
- ]
- }
- }
- ]
- }
- ],
- "placement": {
- "configuration-layout": "default",
- "configs": [
- {
- "config": "druid-common/database_name",
- "subsection-name": "subsection-metadata-storage-row1-col1"
- },
- {
- "config": "druid-common/druid.metadata.storage.type",
- "subsection-name": "subsection-metadata-storage-row1-col1"
- },
- {
- "config": "druid-common/druid.metadata.storage.connector.user",
- "subsection-name": "subsection-metadata-storage-row1-col1"
- },
- {
- "config": "druid-common/druid.metadata.storage.connector.password",
- "subsection-name": "subsection-metadata-storage-row1-col1"
- },
- {
- "config": "druid-common/metastore_hostname",
- "subsection-name": "subsection-metadata-storage-row1-col1"
- },
- {
- "config": "druid-common/druid.metadata.storage.connector.port",
- "subsection-name": "subsection-metadata-storage-row1-col1"
- },
- {
- "config": "druid-common/druid.metadata.storage.connector.connectURI",
- "subsection-name": "subsection-metadata-storage-row1-col1"
- }
- ]
- },
- "widgets": [
- {
- "config": "druid-common/database_name",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "druid-common/druid.metadata.storage.type",
- "widget": {
- "type": "combo"
- }
- },
- {
- "config": "druid-common/druid.metadata.storage.connector.user",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "druid-common/druid.metadata.storage.connector.password",
- "widget": {
- "type": "password"
- }
- },
- {
- "config": "druid-common/metastore_hostname",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "druid-common/druid.metadata.storage.connector.port",
- "widget": {
- "type": "text-field"
- }
- },
- {
- "config": "druid-common/druid.metadata.storage.connector.connectURI",
- "widget": {
- "type": "text-field"
- }
- }
- ]
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml
index f1f099e..07dd6d6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/DRUID/metainfo.xml
@@ -20,9 +20,8 @@
<services>
<service>
<name>DRUID</name>
- <version>0.9.2</version>
- <extends>common-services/DRUID/0.9.2</extends>
- <selection>TECH_PREVIEW</selection>
+ <version>0.10.1</version>
+ <extends>common-services/DRUID/0.10.1</extends>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e61556cc/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
index d8b9fe2..80e9d54 100644
--- a/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
+++ b/ambari-server/src/test/python/stacks/2.6/DRUID/test_druid.py
@@ -31,7 +31,7 @@ from resource_management.core.logger import Logger
@patch("resource_management.libraries.Script.get_tmp_dir", new=MagicMock(return_value=('/var/lib/ambari-agent/tmp')))
@patch.object(functions, "get_stack_version", new=MagicMock(return_value="2.0.0.0-1234"))
class TestDruid(RMFTestCase):
- COMMON_SERVICES_PACKAGE_DIR = "DRUID/0.9.2/package"
+ COMMON_SERVICES_PACKAGE_DIR = "DRUID/0.10.1/package"
STACK_VERSION = "2.6"
DEFAULT_IMMUTABLE_PATHS = ['/apps/hive/warehouse', '/apps/falcon', '/mr-history/done', '/app-logs', '/tmp']