You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2017/03/31 07:22:50 UTC
[43/55] [abbrv] ambari git commit: AMBARI-20617. Display log level,
method name from stack_advisor in ambari-server.log (dlysnichenko)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ef14f56/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index b693f9f..2a1113a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -20,7 +20,6 @@ limitations under the License.
import math
from ambari_commons.str_utils import string_set_equals
-from resource_management.core.logger import Logger
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.get_bare_principal import get_bare_principal
@@ -29,7 +28,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
def __init__(self):
super(HDP25StackAdvisor, self).__init__()
- Logger.initialize_logger()
+ self.initialize_logger("HDP25StackAdvisor")
self.HIVE_INTERACTIVE_SITE = 'hive-interactive-site'
self.YARN_ROOT_DEFAULT_QUEUE_NAME = 'default'
self.AMBARI_MANAGED_LLAP_QUEUE_NAME = 'llap'
@@ -42,11 +41,11 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putOozieEnvProperty = self.putProperty(configurations, "oozie-env", services)
if not "oozie-env" in services["configurations"] :
- Logger.info("No oozie configurations available")
+ self.logger.info("No oozie configurations available")
return
if not "FALCON_SERVER" in clusterData["components"] :
- Logger.info("Falcon is not part of the installation")
+ self.logger.info("Falcon is not part of the installation")
return
falconUser = 'falcon'
@@ -54,35 +53,35 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if "falcon-env" in services["configurations"] :
if "falcon_user" in services["configurations"]["falcon-env"]["properties"] :
falconUser = services["configurations"]["falcon-env"]["properties"]["falcon_user"]
- Logger.info("Falcon user from configuration: %s " % falconUser)
+ self.logger.info("Falcon user from configuration: %s " % falconUser)
- Logger.info("Falcon user : %s" % falconUser)
+ self.logger.info("Falcon user : %s" % falconUser)
oozieUser = 'oozie'
if "oozie_user" \
in services["configurations"]["oozie-env"]["properties"] :
oozieUser = services["configurations"]["oozie-env"]["properties"]["oozie_user"]
- Logger.info("Oozie user from configuration %s" % oozieUser)
+ self.logger.info("Oozie user from configuration %s" % oozieUser)
- Logger.info("Oozie user %s" % oozieUser)
+ self.logger.info("Oozie user %s" % oozieUser)
if "oozie_admin_users" \
in services["configurations"]["oozie-env"]["properties"] :
currentAdminUsers = services["configurations"]["oozie-env"]["properties"]["oozie_admin_users"]
- Logger.info("Oozie admin users from configuration %s" % currentAdminUsers)
+ self.logger.info("Oozie admin users from configuration %s" % currentAdminUsers)
else :
currentAdminUsers = "{0}, oozie-admin".format(oozieUser)
- Logger.info("Setting default oozie admin users to %s" % currentAdminUsers)
+ self.logger.info("Setting default oozie admin users to %s" % currentAdminUsers)
if falconUser in currentAdminUsers :
- Logger.info("Falcon user %s already member of oozie admin users " % falconUser)
+ self.logger.info("Falcon user %s already member of oozie admin users " % falconUser)
return
newAdminUsers = "{0},{1}".format(currentAdminUsers, falconUser)
- Logger.info("new oozie admin users : %s" % newAdminUsers)
+ self.logger.info("new oozie admin users : %s" % newAdminUsers)
services["forced-configurations"].append({"type" : "oozie-env", "name" : "oozie_admin_users"})
putOozieEnvProperty("oozie_admin_users", newAdminUsers)
@@ -158,7 +157,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
auth_type = application_properties['atlas.authentication.method.ldap.type']
auth_ldap_enable = application_properties['atlas.authentication.method.ldap'].lower() == 'true'
- Logger.info("Validating Atlas configs, authentication type: %s" % str(auth_type))
+ self.logger.info("Validating Atlas configs, authentication type: %s" % str(auth_type))
# Required props
ldap_props = {"atlas.authentication.method.ldap.url": "",
@@ -308,7 +307,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
if not capacity_scheduler_properties:
- Logger.warning("Couldn't retrieve 'capacity-scheduler' properties while doing validation checks for Hive Server Interactive.")
+ self.logger.warning("Couldn't retrieve 'capacity-scheduler' properties while doing validation checks for Hive Server Interactive.")
return []
if hsi_site:
@@ -327,7 +326,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
"app to run".format(llap_queue_name, llap_queue_cap_perc, min_reqd_queue_cap_perc)
validationItems.append({"config-name": "hive.llap.daemon.queue.name", "item": self.getErrorItem(errMsg1)})
else:
- Logger.error("Couldn't retrieve '{0}' queue's capacity from 'capacity-scheduler' while doing validation checks for "
+ self.logger.error("Couldn't retrieve '{0}' queue's capacity from 'capacity-scheduler' while doing validation checks for "
"Hive Server Interactive.".format(llap_queue_name))
# Validate that current selected queue in 'hive.llap.daemon.queue.name' state is not STOPPED.
@@ -338,10 +337,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
.format(llap_queue_name, llap_selected_queue_state)
validationItems.append({"config-name": "hive.llap.daemon.queue.name","item": self.getErrorItem(errMsg2)})
else:
- Logger.error("Couldn't retrieve '{0}' queue's state from 'capacity-scheduler' while doing validation checks for "
+ self.logger.error("Couldn't retrieve '{0}' queue's state from 'capacity-scheduler' while doing validation checks for "
"Hive Server Interactive.".format(llap_queue_name))
else:
- Logger.error("Couldn't retrieve 'hive.llap.daemon.queue.name' config from 'hive-interactive-site' while doing "
+ self.logger.error("Couldn't retrieve 'hive.llap.daemon.queue.name' config from 'hive-interactive-site' while doing "
"validation checks for Hive Server Interactive.")
# Validate that 'hive.server2.enable.doAs' config is not set to 'true' for Hive2.
@@ -555,7 +554,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
atlas_rest_address_list = ["{0}://{1}:{2}".format(scheme, hostname, metadata_port) for hostname in atlas_host_names]
atlas_rest_address = ",".join(atlas_rest_address_list)
- Logger.info("Constructing atlas.rest.address=%s" % atlas_rest_address)
+ self.logger.info("Constructing atlas.rest.address=%s" % atlas_rest_address)
return atlas_rest_address
def recommendAtlasConfigurations(self, configurations, clusterData, services, hosts):
@@ -699,15 +698,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putHbaseSiteProperty('hbase.master.ui.readonly', 'true')
phoenix_query_server_hosts = self.get_phoenix_query_server_hosts(services, hosts)
- Logger.debug("Calculated Phoenix Query Server hosts: %s" % str(phoenix_query_server_hosts))
+ self.logger.debug("Calculated Phoenix Query Server hosts: %s" % str(phoenix_query_server_hosts))
if phoenix_query_server_hosts:
- Logger.debug("Attempting to update hadoop.proxyuser.HTTP.hosts with %s" % str(phoenix_query_server_hosts))
+ self.logger.debug("Attempting to update hadoop.proxyuser.HTTP.hosts with %s" % str(phoenix_query_server_hosts))
# The PQS hosts we want to ensure are set
new_value = ','.join(phoenix_query_server_hosts)
# Update the proxyuser setting, deferring to out callback to merge results together
self.put_proxyuser_value("HTTP", new_value, services=services, configurations=configurations, put_function=putCoreSiteProperty)
else:
- Logger.debug("No phoenix query server hosts to update")
+ self.logger.debug("No phoenix query server hosts to update")
else:
putHbaseSiteProperty('hbase.master.ui.readonly', 'false')
@@ -722,7 +721,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
return [host['Hosts']['host_name'] for host in phoenix_query_server_hosts]
def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts):
- Logger.info("DBG: Invoked recommendHiveConfiguration")
+ self.logger.info("DBG: Invoked recommendHiveConfiguration")
super(HDP25StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData, services, hosts)
putHiveInteractiveEnvProperty = self.putProperty(configurations, "hive-interactive-env", services)
putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services)
@@ -747,9 +746,9 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if hive_tez_default_queue:
putHiveInteractiveSiteProperty("hive.server2.tez.default.queues", hive_tez_default_queue)
- Logger.debug("Updated 'hive.server2.tez.default.queues' config : '{0}'".format(hive_tez_default_queue))
+ self.logger.debug("Updated 'hive.server2.tez.default.queues' config : '{0}'".format(hive_tez_default_queue))
else:
- Logger.info("DBG: Setting 'num_llap_nodes' config's READ ONLY attribute as 'True'.")
+ self.logger.info("DBG: Setting 'num_llap_nodes' config's READ ONLY attribute as 'True'.")
putHiveInteractiveEnvProperty('enable_hive_interactive', 'false')
putHiveInteractiveEnvPropertyAttribute("num_llap_nodes", "read_only", "true")
@@ -810,14 +809,14 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
Note: All memory calculations are in MB, unless specified otherwise.
"""
- Logger.info("DBG: Entered updateLlapConfigs");
+ self.logger.info("DBG: Entered updateLlapConfigs");
# Determine if we entered here during cluster creation.
operation = getUserOperationContext(services, "operation")
is_cluster_create_opr = False
if operation == self.CLUSTER_CREATE_OPERATION:
is_cluster_create_opr = True
- Logger.info("Is cluster create operation ? = {0}".format(is_cluster_create_opr))
+ self.logger.info("Is cluster create operation ? = {0}".format(is_cluster_create_opr))
putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services)
putHiveInteractiveSitePropertyAttribute = self.putPropertyAttribute(configurations, self.HIVE_INTERACTIVE_SITE)
@@ -852,9 +851,9 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if capacity_scheduler_properties:
# Get all leaf queues.
leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties)
- Logger.info("YARN leaf Queues = {0}".format(leafQueueNames))
+ self.logger.info("YARN leaf Queues = {0}".format(leafQueueNames))
if len(leafQueueNames) == 0:
- Logger.error("Queue(s) couldn't be retrieved from capacity-scheduler.")
+ self.logger.error("Queue(s) couldn't be retrieved from capacity-scheduler.")
return
# Check if it's 1st invocation after enabling Hive Server Interactive (config: enable_hive_interactive).
@@ -873,20 +872,20 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', first_leaf_queue)
putHiveInteractiveSiteProperty('hive.server2.tez.default.queues', first_leaf_queue)
llap_named_queue_selected_in_curr_invocation = False
- Logger.info("DBG: llap_named_queue_selected_in_curr_invocation = {0}".format(llap_named_queue_selected_in_curr_invocation))
+ self.logger.info("DBG: llap_named_queue_selected_in_curr_invocation = {0}".format(llap_named_queue_selected_in_curr_invocation))
if (len(leafQueueNames) == 2 and (llap_daemon_selected_queue_name and llap_daemon_selected_queue_name == llap_queue_name) or
llap_named_queue_selected_in_curr_invocation) or \
(len(leafQueueNames) == 1 and llap_daemon_selected_queue_name == 'default' and llap_named_queue_selected_in_curr_invocation):
- Logger.info("DBG: Setting 'num_llap_nodes' config's READ ONLY attribute as 'False'.")
+ self.logger.info("DBG: Setting 'num_llap_nodes' config's READ ONLY attribute as 'False'.")
putHiveInteractiveEnvPropertyAttribute("num_llap_nodes", "read_only", "false")
selected_queue_is_ambari_managed_llap = True
- Logger.info("DBG: Selected YARN queue for LLAP is : '{0}'. Current YARN queues : {1}. Setting 'Number of LLAP nodes' "
+ self.logger.info("DBG: Selected YARN queue for LLAP is : '{0}'. Current YARN queues : {1}. Setting 'Number of LLAP nodes' "
"slider visibility to 'True'".format(llap_queue_name, list(leafQueueNames)))
else:
- Logger.info("DBG: Setting 'num_llap_nodes' config's READ ONLY attribute as 'True'.")
+ self.logger.info("DBG: Setting 'num_llap_nodes' config's READ ONLY attribute as 'True'.")
putHiveInteractiveEnvPropertyAttribute("num_llap_nodes", "read_only", "true")
- Logger.info("Selected YARN queue for LLAP is : '{0}'. Current YARN queues : {1}. Setting 'Number of LLAP nodes' "
+ self.logger.info("Selected YARN queue for LLAP is : '{0}'. Current YARN queues : {1}. Setting 'Number of LLAP nodes' "
"visibility to 'False'.".format(llap_daemon_selected_queue_name, list(leafQueueNames)))
selected_queue_is_ambari_managed_llap = False
@@ -896,17 +895,17 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if llap_daemon_selected_queue_name:
llap_selected_queue_state = self.__getQueueStateFromCapacityScheduler(capacity_scheduler_properties, llap_daemon_selected_queue_name)
if llap_selected_queue_state is None or llap_selected_queue_state == "STOPPED":
- Logger.error("Selected LLAP app queue '{0}' current state is : '{1}'. Setting LLAP configs to default "
+ self.logger.error("Selected LLAP app queue '{0}' current state is : '{1}'. Setting LLAP configs to default "
"values.".format(llap_daemon_selected_queue_name, llap_selected_queue_state))
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
else:
- Logger.error("Retrieved LLAP app queue name is : '{0}'. Setting LLAP configs to default values."
+ self.logger.error("Retrieved LLAP app queue name is : '{0}'. Setting LLAP configs to default values."
.format(llap_daemon_selected_queue_name))
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
else:
- Logger.error("Couldn't retrieve 'capacity-scheduler' properties while doing YARN queue adjustment for Hive Server Interactive."
+ self.logger.error("Couldn't retrieve 'capacity-scheduler' properties while doing YARN queue adjustment for Hive Server Interactive."
" Not calculating LLAP configs.")
return
@@ -928,16 +927,16 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if not changed_configs_in_hive_int_env and not llap_concurrency_in_changed_configs and \
not llap_daemon_queue_in_changed_configs and services["changed-configurations"]:
- Logger.info("DBG: LLAP parameters not modified. Not adjusting LLAP configs.")
- Logger.info("DBG: Current 'changed-configuration' received is : {0}".format(services["changed-configurations"]))
+ self.logger.info("DBG: LLAP parameters not modified. Not adjusting LLAP configs.")
+ self.logger.info("DBG: Current 'changed-configuration' received is : {0}".format(services["changed-configurations"]))
return
- Logger.info("\nDBG: Performing LLAP config calculations ......")
+ self.logger.info("\nDBG: Performing LLAP config calculations ......")
node_manager_host_list = self.getHostsForComponent(services, "YARN", "NODEMANAGER")
node_manager_cnt = len(node_manager_host_list)
yarn_nm_mem_in_mb = self.get_yarn_nm_mem_in_mb(services, configurations)
total_cluster_capacity = node_manager_cnt * yarn_nm_mem_in_mb
- Logger.info("DBG: Calculated total_cluster_capacity : {0}, using following : node_manager_cnt : {1}, "
+ self.logger.info("DBG: Calculated total_cluster_capacity : {0}, using following : node_manager_cnt : {1}, "
"yarn_nm_mem_in_mb : {2}".format(total_cluster_capacity, node_manager_cnt, yarn_nm_mem_in_mb))
yarn_min_container_size = float(self.get_yarn_min_container_size(services, configurations))
tez_am_container_size = self.calculate_tez_am_container_size(services, long(total_cluster_capacity), is_cluster_create_opr,
@@ -949,7 +948,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
else:
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
- Logger.info("DBG Calculated normalized_tez_am_container_size : {0}, using following : tez_am_container_size : {1}, "
+ self.logger.info("DBG Calculated normalized_tez_am_container_size : {0}, using following : tez_am_container_size : {1}, "
"total_cluster_capacity : {2}".format(normalized_tez_am_container_size, tez_am_container_size,
total_cluster_capacity))
@@ -957,7 +956,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
yarn_nm_mem_in_mb_normalized = self._normalizeDown(yarn_nm_mem_in_mb, yarn_min_container_size)
mem_per_thread_for_llap = float(self.calculate_mem_per_thread_for_llap(services, yarn_nm_mem_in_mb_normalized, cpu_per_nm_host,
is_cluster_create_opr, changed_configs_has_enable_hive_int))
- Logger.info("DBG: Calculated mem_per_thread_for_llap : {0}, using following: yarn_nm_mem_in_mb_normalized : {1}, "
+ self.logger.info("DBG: Calculated mem_per_thread_for_llap : {0}, using following: yarn_nm_mem_in_mb_normalized : {1}, "
"cpu_per_nm_host : {2}".format(mem_per_thread_for_llap, yarn_nm_mem_in_mb_normalized, cpu_per_nm_host))
@@ -968,48 +967,48 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
# Get calculated value for Slider AM container Size
slider_am_container_size = self._normalizeUp(self.calculate_slider_am_size(yarn_min_container_size),
yarn_min_container_size)
- Logger.info("DBG: Calculated 'slider_am_container_size' : {0}, using following: yarn_min_container_size : "
+ self.logger.info("DBG: Calculated 'slider_am_container_size' : {0}, using following: yarn_min_container_size : "
"{1}".format(slider_am_container_size, yarn_min_container_size))
min_memory_required = normalized_tez_am_container_size + slider_am_container_size + self._normalizeUp(mem_per_thread_for_llap, yarn_min_container_size)
- Logger.info("DBG: Calculated 'min_memory_required': {0} using following : slider_am_container_size: {1}, "
+ self.logger.info("DBG: Calculated 'min_memory_required': {0} using following : slider_am_container_size: {1}, "
"normalized_tez_am_container_size : {2}, mem_per_thread_for_llap : {3}, yarn_min_container_size : "
"{4}".format(min_memory_required, slider_am_container_size, normalized_tez_am_container_size, mem_per_thread_for_llap, yarn_min_container_size))
min_nodes_required = int(math.ceil( min_memory_required / yarn_nm_mem_in_mb_normalized))
- Logger.info("DBG: Calculated 'min_node_required': {0}, using following : min_memory_required : {1}, yarn_nm_mem_in_mb_normalized "
+ self.logger.info("DBG: Calculated 'min_node_required': {0}, using following : min_memory_required : {1}, yarn_nm_mem_in_mb_normalized "
": {2}".format(min_nodes_required, min_memory_required, yarn_nm_mem_in_mb_normalized))
if min_nodes_required > node_manager_cnt:
- Logger.warning("ERROR: Not enough memory/nodes to run LLAP");
+ self.logger.warning("ERROR: Not enough memory/nodes to run LLAP");
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
mem_per_thread_for_llap = float(mem_per_thread_for_llap)
- Logger.info("DBG: selected_queue_is_ambari_managed_llap = {0}".format(selected_queue_is_ambari_managed_llap))
+ self.logger.info("DBG: selected_queue_is_ambari_managed_llap = {0}".format(selected_queue_is_ambari_managed_llap))
if not selected_queue_is_ambari_managed_llap:
llap_daemon_selected_queue_cap = self.__getSelectedQueueTotalCap(capacity_scheduler_properties, llap_daemon_selected_queue_name, total_cluster_capacity)
if llap_daemon_selected_queue_cap <= 0:
- Logger.warning("'{0}' queue capacity percentage retrieved = {1}. Expected > 0.".format(
+ self.logger.warning("'{0}' queue capacity percentage retrieved = {1}. Expected > 0.".format(
llap_daemon_selected_queue_name, llap_daemon_selected_queue_cap))
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
total_llap_mem_normalized = self._normalizeDown(llap_daemon_selected_queue_cap, yarn_min_container_size)
- Logger.info("DBG: Calculated '{0}' queue available capacity : {1}, using following: llap_daemon_selected_queue_cap : {2}, "
+ self.logger.info("DBG: Calculated '{0}' queue available capacity : {1}, using following: llap_daemon_selected_queue_cap : {2}, "
"yarn_min_container_size : {3}".format(llap_daemon_selected_queue_name, total_llap_mem_normalized,
llap_daemon_selected_queue_cap, yarn_min_container_size))
'''Rounding up numNodes so that we run more daemons, and utilitze more CPUs. The rest of the calcaulations will take care of cutting this down if required'''
num_llap_nodes_requested = math.ceil(total_llap_mem_normalized / yarn_nm_mem_in_mb_normalized)
- Logger.info("DBG: Calculated 'num_llap_nodes_requested' : {0}, using following: total_llap_mem_normalized : {1}, "
+ self.logger.info("DBG: Calculated 'num_llap_nodes_requested' : {0}, using following: total_llap_mem_normalized : {1}, "
"yarn_nm_mem_in_mb_normalized : {2}".format(num_llap_nodes_requested, total_llap_mem_normalized, yarn_nm_mem_in_mb_normalized))
# Pouplate the 'num_llap_nodes_requested' in config 'num_llap_nodes', a read only config for non-Ambari managed queue case.
putHiveInteractiveEnvProperty('num_llap_nodes', num_llap_nodes_requested)
- Logger.info("Setting config 'num_llap_nodes' as : {0}".format(num_llap_nodes_requested))
+ self.logger.info("Setting config 'num_llap_nodes' as : {0}".format(num_llap_nodes_requested))
queue_am_fraction_perc = float(self.__getQueueAmFractionFromCapacityScheduler(capacity_scheduler_properties, llap_daemon_selected_queue_name))
hive_tez_am_cap_available = queue_am_fraction_perc * total_llap_mem_normalized
- Logger.info("DBG: Calculated 'hive_tez_am_cap_available' : {0}, using following: queue_am_fraction_perc : {1}, "
+ self.logger.info("DBG: Calculated 'hive_tez_am_cap_available' : {0}, using following: queue_am_fraction_perc : {1}, "
"total_llap_mem_normalized : {2}".format(hive_tez_am_cap_available, queue_am_fraction_perc, total_llap_mem_normalized))
else: # Ambari managed 'llap' named queue at root level.
# Set 'num_llap_nodes_requested' for 1st invocation, as it gets passed as 1 otherwise, read from config.
@@ -1022,34 +1021,34 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
else:
num_llap_nodes_requested = self.get_num_llap_nodes(services, configurations) #Input
total_llap_mem = num_llap_nodes_requested * yarn_nm_mem_in_mb_normalized
- Logger.info("DBG: Calculated 'total_llap_mem' : {0}, using following: num_llap_nodes_requested : {1}, "
+ self.logger.info("DBG: Calculated 'total_llap_mem' : {0}, using following: num_llap_nodes_requested : {1}, "
"yarn_nm_mem_in_mb_normalized : {2}".format(total_llap_mem, num_llap_nodes_requested, yarn_nm_mem_in_mb_normalized))
total_llap_mem_normalized = float(self._normalizeDown(total_llap_mem, yarn_min_container_size))
- Logger.info("DBG: Calculated 'total_llap_mem_normalized' : {0}, using following: total_llap_mem : {1}, "
+ self.logger.info("DBG: Calculated 'total_llap_mem_normalized' : {0}, using following: total_llap_mem : {1}, "
"yarn_min_container_size : {2}".format(total_llap_mem_normalized, total_llap_mem, yarn_min_container_size))
# What percent is 'total_llap_mem' of 'total_cluster_capacity' ?
llap_named_queue_cap_fraction = math.ceil(total_llap_mem_normalized / total_cluster_capacity * 100)
- Logger.info("DBG: Calculated '{0}' queue capacity percent = {1}.".format(llap_queue_name, llap_named_queue_cap_fraction))
+ self.logger.info("DBG: Calculated '{0}' queue capacity percent = {1}.".format(llap_queue_name, llap_named_queue_cap_fraction))
if llap_named_queue_cap_fraction > 100:
- Logger.warning("Calculated '{0}' queue size = {1}. Cannot be > 100.".format(llap_queue_name, llap_named_queue_cap_fraction))
+ self.logger.warning("Calculated '{0}' queue size = {1}. Cannot be > 100.".format(llap_queue_name, llap_named_queue_cap_fraction))
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
# Adjust capacity scheduler for the 'llap' named queue.
self.checkAndManageLlapQueue(services, configurations, hosts, llap_queue_name, llap_named_queue_cap_fraction)
hive_tez_am_cap_available = total_llap_mem_normalized
- Logger.info("DBG: hive_tez_am_cap_available : {0}".format(hive_tez_am_cap_available))
+ self.logger.info("DBG: hive_tez_am_cap_available : {0}".format(hive_tez_am_cap_available))
# Common calculations now, irrespective of the queue selected.
llap_mem_for_tezAm_and_daemons = total_llap_mem_normalized - slider_am_container_size
- Logger.info("DBG: Calculated 'llap_mem_for_tezAm_and_daemons' : {0}, using following : total_llap_mem_normalized : {1}, "
+ self.logger.info("DBG: Calculated 'llap_mem_for_tezAm_and_daemons' : {0}, using following : total_llap_mem_normalized : {1}, "
"slider_am_container_size : {2}".format(llap_mem_for_tezAm_and_daemons, total_llap_mem_normalized, slider_am_container_size))
if llap_mem_for_tezAm_and_daemons < 2 * yarn_min_container_size:
- Logger.warning("Not enough capacity available on the cluster to run LLAP")
+ self.logger.warning("Not enough capacity available on the cluster to run LLAP")
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
@@ -1059,11 +1058,11 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
# Read 'hive.server2.tez.sessions.per.default.queue' prop if it's in changed-configs, else calculate it.
if not llap_concurrency_in_changed_configs:
if max_executors_per_node <= 0:
- Logger.warning("Calculated 'max_executors_per_node' = {0}. Expected value >= 1.".format(max_executors_per_node))
+ self.logger.warning("Calculated 'max_executors_per_node' = {0}. Expected value >= 1.".format(max_executors_per_node))
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
- Logger.info("DBG: Calculated 'max_executors_per_node' : {0}, using following: yarn_nm_mem_in_mb_normalized : {1}, cpu_per_nm_host : {2}, "
+ self.logger.info("DBG: Calculated 'max_executors_per_node' : {0}, using following: yarn_nm_mem_in_mb_normalized : {1}, cpu_per_nm_host : {2}, "
"mem_per_thread_for_llap: {3}".format(max_executors_per_node, yarn_nm_mem_in_mb_normalized, cpu_per_nm_host, mem_per_thread_for_llap))
# Default 1 AM for every 20 executor threads.
@@ -1071,122 +1070,122 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
# making use of total memory. However, it's possible that total memory will not be used - and the numExecutors is
# instead limited by #CPUs. Use maxPerNode to factor this in.
llap_concurreny_limit = min(math.floor(max_executors_per_node * num_llap_nodes_requested / DEFAULT_EXECUTOR_TO_AM_RATIO), MAX_CONCURRENT_QUERIES)
- Logger.info("DBG: Calculated 'llap_concurreny_limit' : {0}, using following : max_executors_per_node : {1}, num_llap_nodes_requested : {2}, DEFAULT_EXECUTOR_TO_AM_RATIO "
+ self.logger.info("DBG: Calculated 'llap_concurreny_limit' : {0}, using following : max_executors_per_node : {1}, num_llap_nodes_requested : {2}, DEFAULT_EXECUTOR_TO_AM_RATIO "
": {3}, MAX_CONCURRENT_QUERIES : {4}".format(llap_concurreny_limit, max_executors_per_node, num_llap_nodes_requested, DEFAULT_EXECUTOR_TO_AM_RATIO, MAX_CONCURRENT_QUERIES))
llap_concurrency = min(llap_concurreny_limit, math.floor(llap_mem_for_tezAm_and_daemons / (DEFAULT_EXECUTOR_TO_AM_RATIO * mem_per_thread_for_llap + normalized_tez_am_container_size)))
- Logger.info("DBG: Calculated 'llap_concurrency' : {0}, using following : llap_concurreny_limit : {1}, llap_mem_for_tezAm_and_daemons : "
+ self.logger.info("DBG: Calculated 'llap_concurrency' : {0}, using following : llap_concurreny_limit : {1}, llap_mem_for_tezAm_and_daemons : "
"{2}, DEFAULT_EXECUTOR_TO_AM_RATIO : {3}, mem_per_thread_for_llap : {4}, normalized_tez_am_container_size : "
"{5}".format(llap_concurrency, llap_concurreny_limit, llap_mem_for_tezAm_and_daemons, DEFAULT_EXECUTOR_TO_AM_RATIO,
mem_per_thread_for_llap, normalized_tez_am_container_size))
if llap_concurrency == 0:
llap_concurrency = 1
- Logger.info("DBG: Readjusted 'llap_concurrency' to : 1. Earlier calculated value : 0")
+ self.logger.info("DBG: Readjusted 'llap_concurrency' to : 1. Earlier calculated value : 0")
if llap_concurrency * normalized_tez_am_container_size > hive_tez_am_cap_available:
llap_concurrency = long(math.floor(hive_tez_am_cap_available / normalized_tez_am_container_size))
- Logger.info("DBG: Readjusted 'llap_concurrency' to : {0}, as llap_concurrency({1}) * normalized_tez_am_container_size({2}) > hive_tez_am_cap_available({3}))"
+ self.logger.info("DBG: Readjusted 'llap_concurrency' to : {0}, as llap_concurrency({1}) * normalized_tez_am_container_size({2}) > hive_tez_am_cap_available({3}))"
.format(llap_concurrency, llap_concurrency, normalized_tez_am_container_size, hive_tez_am_cap_available))
if llap_concurrency <= 0:
- Logger.warning("DBG: Calculated 'LLAP Concurrent Queries' = {0}. Expected value >= 1.".format(llap_concurrency))
+ self.logger.warning("DBG: Calculated 'LLAP Concurrent Queries' = {0}. Expected value >= 1.".format(llap_concurrency))
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
- Logger.info("DBG: Adjusted 'llap_concurrency' : {0}, using following: hive_tez_am_cap_available : {1}, normalized_tez_am_container_size: "
+ self.logger.info("DBG: Adjusted 'llap_concurrency' : {0}, using following: hive_tez_am_cap_available : {1}, normalized_tez_am_container_size: "
"{2}".format(llap_concurrency, hive_tez_am_cap_available, normalized_tez_am_container_size))
else:
# Read current value
if 'hive.server2.tez.sessions.per.default.queue' in hsi_site:
llap_concurrency = long(hsi_site['hive.server2.tez.sessions.per.default.queue'])
if llap_concurrency <= 0:
- Logger.warning("'hive.server2.tez.sessions.per.default.queue' current value : {0}. Expected value : >= 1".format(llap_concurrency))
+ self.logger.warning("'hive.server2.tez.sessions.per.default.queue' current value : {0}. Expected value : >= 1".format(llap_concurrency))
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
- Logger.info("DBG: Read 'llap_concurrency' : {0}".format(llap_concurrency ))
+ self.logger.info("DBG: Read 'llap_concurrency' : {0}".format(llap_concurrency ))
else:
llap_concurrency = 1
- Logger.warning("Couldn't retrieve Hive Server interactive's 'hive.server2.tez.sessions.per.default.queue' config. Setting default value 1.")
+ self.logger.warning("Couldn't retrieve Hive Server interactive's 'hive.server2.tez.sessions.per.default.queue' config. Setting default value 1.")
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
# Calculate 'Max LLAP Consurrency', irrespective of whether 'llap_concurrency' was read or calculated.
max_llap_concurreny_limit = min(math.floor(max_executors_per_node * num_llap_nodes_requested / MIN_EXECUTOR_TO_AM_RATIO), MAX_CONCURRENT_QUERIES)
- Logger.info("DBG: Calculated 'max_llap_concurreny_limit' : {0}, using following : max_executors_per_node : {1}, num_llap_nodes_requested "
+ self.logger.info("DBG: Calculated 'max_llap_concurreny_limit' : {0}, using following : max_executors_per_node : {1}, num_llap_nodes_requested "
": {2}, MIN_EXECUTOR_TO_AM_RATIO : {3}, MAX_CONCURRENT_QUERIES : {4}".format(max_llap_concurreny_limit, max_executors_per_node,
num_llap_nodes_requested, MIN_EXECUTOR_TO_AM_RATIO,
MAX_CONCURRENT_QUERIES))
max_llap_concurreny = long(min(max_llap_concurreny_limit, math.floor(llap_mem_for_tezAm_and_daemons / (MIN_EXECUTOR_TO_AM_RATIO *
mem_per_thread_for_llap + normalized_tez_am_container_size))))
- Logger.info("DBG: Calculated 'max_llap_concurreny' : {0}, using following : max_llap_concurreny_limit : {1}, llap_mem_for_tezAm_and_daemons : "
+ self.logger.info("DBG: Calculated 'max_llap_concurreny' : {0}, using following : max_llap_concurreny_limit : {1}, llap_mem_for_tezAm_and_daemons : "
"{2}, MIN_EXECUTOR_TO_AM_RATIO : {3}, mem_per_thread_for_llap : {4}, normalized_tez_am_container_size : "
"{5}".format(max_llap_concurreny, max_llap_concurreny_limit, llap_mem_for_tezAm_and_daemons, MIN_EXECUTOR_TO_AM_RATIO,
mem_per_thread_for_llap, normalized_tez_am_container_size))
if int(max_llap_concurreny) < MAX_CONCURRENT_QUERIES_SMALL_CLUSTERS:
- Logger.info("DBG: Adjusting 'max_llap_concurreny' from {0} to {1}".format(max_llap_concurreny, MAX_CONCURRENT_QUERIES_SMALL_CLUSTERS))
+ self.logger.info("DBG: Adjusting 'max_llap_concurreny' from {0} to {1}".format(max_llap_concurreny, MAX_CONCURRENT_QUERIES_SMALL_CLUSTERS))
max_llap_concurreny = MAX_CONCURRENT_QUERIES_SMALL_CLUSTERS
if (max_llap_concurreny * normalized_tez_am_container_size) > hive_tez_am_cap_available:
max_llap_concurreny = math.floor(hive_tez_am_cap_available / normalized_tez_am_container_size)
if max_llap_concurreny <= 0:
- Logger.warning("Calculated 'Max. LLAP Concurrent Queries' = {0}. Expected value > 1".format(max_llap_concurreny))
+ self.logger.warning("Calculated 'Max. LLAP Concurrent Queries' = {0}. Expected value > 1".format(max_llap_concurreny))
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
- Logger.info("DBG: Adjusted 'max_llap_concurreny' : {0}, using following: hive_tez_am_cap_available : {1}, normalized_tez_am_container_size: "
+ self.logger.info("DBG: Adjusted 'max_llap_concurreny' : {0}, using following: hive_tez_am_cap_available : {1}, normalized_tez_am_container_size: "
"{2}".format(max_llap_concurreny, hive_tez_am_cap_available, normalized_tez_am_container_size))
# Calculate value for 'num_llap_nodes', an across cluster config.
tez_am_memory_required = llap_concurrency * normalized_tez_am_container_size
- Logger.info("DBG: Calculated 'tez_am_memory_required' : {0}, using following : llap_concurrency : {1}, normalized_tez_am_container_size : "
+ self.logger.info("DBG: Calculated 'tez_am_memory_required' : {0}, using following : llap_concurrency : {1}, normalized_tez_am_container_size : "
"{2}".format(tez_am_memory_required, llap_concurrency, normalized_tez_am_container_size))
llap_mem_daemon_size = llap_mem_for_tezAm_and_daemons - tez_am_memory_required
if llap_mem_daemon_size < yarn_min_container_size:
- Logger.warning("Calculated 'LLAP Daemon Size = {0}'. Expected >= 'YARN Minimum Container Size' ({1})'".format(
+ self.logger.warning("Calculated 'LLAP Daemon Size = {0}'. Expected >= 'YARN Minimum Container Size' ({1})'".format(
llap_mem_daemon_size, yarn_min_container_size))
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
if llap_mem_daemon_size < mem_per_thread_for_llap or llap_mem_daemon_size < yarn_min_container_size:
- Logger.warning("Not enough memory available for executors.")
+ self.logger.warning("Not enough memory available for executors.")
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
- Logger.info("DBG: Calculated 'llap_mem_daemon_size' : {0}, using following : llap_mem_for_tezAm_and_daemons : {1}, tez_am_memory_required : "
+ self.logger.info("DBG: Calculated 'llap_mem_daemon_size' : {0}, using following : llap_mem_for_tezAm_and_daemons : {1}, tez_am_memory_required : "
"{2}".format(llap_mem_daemon_size, llap_mem_for_tezAm_and_daemons, tez_am_memory_required))
llap_daemon_mem_per_node = self._normalizeDown(llap_mem_daemon_size / num_llap_nodes_requested, yarn_min_container_size)
- Logger.info("DBG: Calculated 'llap_daemon_mem_per_node' : {0}, using following : llap_mem_daemon_size : {1}, num_llap_nodes_requested : {2}, "
+ self.logger.info("DBG: Calculated 'llap_daemon_mem_per_node' : {0}, using following : llap_mem_daemon_size : {1}, num_llap_nodes_requested : {2}, "
"yarn_min_container_size: {3}".format(llap_daemon_mem_per_node, llap_mem_daemon_size, num_llap_nodes_requested, yarn_min_container_size))
if llap_daemon_mem_per_node == 0:
# Small cluster. No capacity left on a node after running AMs.
llap_daemon_mem_per_node = self._normalizeUp(mem_per_thread_for_llap, yarn_min_container_size)
num_llap_nodes = math.floor(llap_mem_daemon_size / llap_daemon_mem_per_node)
- Logger.info("DBG: 'llap_daemon_mem_per_node' : 0, adjusted 'llap_daemon_mem_per_node' : {0}, 'num_llap_nodes' : {1}, using following: llap_mem_daemon_size : {2}, "
+ self.logger.info("DBG: 'llap_daemon_mem_per_node' : 0, adjusted 'llap_daemon_mem_per_node' : {0}, 'num_llap_nodes' : {1}, using following: llap_mem_daemon_size : {2}, "
"mem_per_thread_for_llap : {3}".format(llap_daemon_mem_per_node, num_llap_nodes, llap_mem_daemon_size, mem_per_thread_for_llap))
elif llap_daemon_mem_per_node < mem_per_thread_for_llap:
# Previously computed value of memory per thread may be too high. Cut the number of nodes. (Alternately reduce memory per node)
llap_daemon_mem_per_node = mem_per_thread_for_llap
num_llap_nodes = math.floor(llap_mem_daemon_size / mem_per_thread_for_llap)
- Logger.info("DBG: 'llap_daemon_mem_per_node'({0}) < mem_per_thread_for_llap({1}), adjusted 'llap_daemon_mem_per_node' "
+ self.logger.info("DBG: 'llap_daemon_mem_per_node'({0}) < mem_per_thread_for_llap({1}), adjusted 'llap_daemon_mem_per_node' "
": {2}".format(llap_daemon_mem_per_node, mem_per_thread_for_llap, llap_daemon_mem_per_node))
else:
# All good. We have a proper value for memoryPerNode.
num_llap_nodes = num_llap_nodes_requested
- Logger.info("DBG: num_llap_nodes : {0}".format(num_llap_nodes))
+ self.logger.info("DBG: num_llap_nodes : {0}".format(num_llap_nodes))
num_executors_per_node_max = self.get_max_executors_per_node(yarn_nm_mem_in_mb_normalized, cpu_per_nm_host, mem_per_thread_for_llap)
if num_executors_per_node_max < 1:
- Logger.warning("Calculated 'Max. Executors per Node' = {0}. Expected values >= 1.".format(num_executors_per_node_max))
+ self.logger.warning("Calculated 'Max. Executors per Node' = {0}. Expected values >= 1.".format(num_executors_per_node_max))
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
- Logger.info("DBG: Calculated 'num_executors_per_node_max' : {0}, using following : yarn_nm_mem_in_mb_normalized : {1}, cpu_per_nm_host : {2}, "
+ self.logger.info("DBG: Calculated 'num_executors_per_node_max' : {0}, using following : yarn_nm_mem_in_mb_normalized : {1}, cpu_per_nm_host : {2}, "
"mem_per_thread_for_llap: {3}".format(num_executors_per_node_max, yarn_nm_mem_in_mb_normalized, cpu_per_nm_host, mem_per_thread_for_llap))
# NumExecutorsPerNode is not necessarily max - since some capacity would have been reserved for AMs, if this value were based on mem.
num_executors_per_node = min(math.floor(llap_daemon_mem_per_node / mem_per_thread_for_llap), num_executors_per_node_max)
if num_executors_per_node <= 0:
- Logger.warning("Calculated 'Number of Executors Per Node' = {0}. Expected value >= 1".format(num_executors_per_node))
+ self.logger.warning("Calculated 'Number of Executors Per Node' = {0}. Expected value >= 1".format(num_executors_per_node))
self.recommendDefaultLlapConfiguration(configurations, services, hosts)
return
- Logger.info("DBG: Calculated 'num_executors_per_node' : {0}, using following : llap_daemon_mem_per_node : {1}, num_executors_per_node_max : {2}, "
+ self.logger.info("DBG: Calculated 'num_executors_per_node' : {0}, using following : llap_daemon_mem_per_node : {1}, num_executors_per_node_max : {2}, "
"mem_per_thread_for_llap: {3}".format(num_executors_per_node, llap_daemon_mem_per_node, num_executors_per_node_max, mem_per_thread_for_llap))
# Now figure out how much of the memory will be used by the executors, and how much will be used by the cache.
@@ -1200,7 +1199,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
# Calculate value for prop 'llap_heap_size'
llap_xmx = max(total_mem_for_executors_per_node * 0.8, total_mem_for_executors_per_node - self.get_llap_headroom_space(services, configurations))
- Logger.info("DBG: Calculated llap_app_heap_size : {0}, using following : total_mem_for_executors : {1}".format(llap_xmx, total_mem_for_executors_per_node))
+ self.logger.info("DBG: Calculated llap_app_heap_size : {0}, using following : total_mem_for_executors : {1}".format(llap_xmx, total_mem_for_executors_per_node))
# Calculate 'hive_heapsize' for Hive2/HiveServer2 (HSI)
hive_server_interactive_heapsize = None
@@ -1211,16 +1210,16 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if hive_server_interactive_hosts is not None and len(hive_server_interactive_hosts) > 0:
host_mem = long(hive_server_interactive_hosts[0]["Hosts"]["total_mem"])
hive_server_interactive_heapsize = min(max(2048.0, 400.0*llap_concurrency), 3.0/8 * host_mem)
- Logger.info("DBG: Calculated 'hive_server_interactive_heapsize' : {0}, using following : llap_concurrency : {1}, host_mem : "
+ self.logger.info("DBG: Calculated 'hive_server_interactive_heapsize' : {0}, using following : llap_concurrency : {1}, host_mem : "
"{2}".format(hive_server_interactive_heapsize, llap_concurrency, host_mem))
# Done with calculations, updating calculated configs.
- Logger.info("DBG: Applying the calculated values....")
+ self.logger.info("DBG: Applying the calculated values....")
if is_cluster_create_opr or changed_configs_has_enable_hive_int:
normalized_tez_am_container_size = long(normalized_tez_am_container_size)
putTezInteractiveSiteProperty('tez.am.resource.memory.mb', normalized_tez_am_container_size)
- Logger.info("DBG: Setting 'tez.am.resource.memory.mb' config value as : {0}".format(normalized_tez_am_container_size))
+ self.logger.info("DBG: Setting 'tez.am.resource.memory.mb' config value as : {0}".format(normalized_tez_am_container_size))
if not llap_concurrency_in_changed_configs:
min_llap_concurrency = 1
@@ -1236,16 +1235,16 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putHiveInteractiveEnvPropertyAttribute('num_llap_nodes', "maximum", node_manager_cnt)
#TODO A single value is not being set for numNodes in case of a custom queue. Also the attribute is set to non-visible, so the UI likely ends up using an old cached value
if (num_llap_nodes != num_llap_nodes_requested):
- Logger.info("DBG: User requested num_llap_nodes : {0}, but used/adjusted value for calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes))
+ self.logger.info("DBG: User requested num_llap_nodes : {0}, but used/adjusted value for calculations is : {1}".format(num_llap_nodes_requested, num_llap_nodes))
else:
- Logger.info("DBG: Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested))
+ self.logger.info("DBG: Used num_llap_nodes for calculations : {0}".format(num_llap_nodes_requested))
# Safeguard for not adding "num_llap_nodes_for_llap_daemons" if it doesnt exist in hive-interactive-site.
# This can happen if we upgrade from Ambari 2.4 (with HDP 2.5) to Ambari 2.5, as this config is from 2.6 stack onwards only.
if "hive-interactive-env" in services["configurations"] and \
"num_llap_nodes_for_llap_daemons" in services["configurations"]["hive-interactive-env"]["properties"]:
putHiveInteractiveEnvProperty('num_llap_nodes_for_llap_daemons', num_llap_nodes)
- Logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes))
+ self.logger.info("DBG: Setting config 'num_llap_nodes_for_llap_daemons' as : {0}".format(num_llap_nodes))
llap_container_size = long(llap_daemon_mem_per_node)
putHiveInteractiveSiteProperty('hive.llap.daemon.yarn.container.mb', llap_container_size)
@@ -1255,7 +1254,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
if is_cluster_create_opr or changed_configs_has_enable_hive_int:
mem_per_thread_for_llap = long(mem_per_thread_for_llap)
putHiveInteractiveSiteProperty('hive.tez.container.size', mem_per_thread_for_llap)
- Logger.info("DBG: Setting 'hive.tez.container.size' config value as : {0}".format(mem_per_thread_for_llap))
+ self.logger.info("DBG: Setting 'hive.tez.container.size' config value as : {0}".format(mem_per_thread_for_llap))
putTezInteractiveSiteProperty('tez.runtime.io.sort.mb', tez_runtime_io_sort_mb)
@@ -1267,7 +1266,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putHiveInteractiveSiteProperty('hive.auto.convert.join.noconditionaltask.size', hive_auto_convert_join_noconditionaltask_size)
num_executors_per_node = long(num_executors_per_node)
- Logger.info("DBG: Putting num_executors_per_node as {0}".format(num_executors_per_node))
+ self.logger.info("DBG: Putting num_executors_per_node as {0}".format(num_executors_per_node))
putHiveInteractiveSiteProperty('hive.llap.daemon.num.executors', num_executors_per_node)
putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "minimum", 1)
putHiveInteractiveSitePropertyAttribute('hive.llap.daemon.num.executors', "maximum", long(num_executors_per_node_max))
@@ -1287,11 +1286,11 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
putHiveInteractiveEnvProperty('llap_heap_size', long(llap_xmx))
putHiveInteractiveEnvProperty('slider_am_container_mb', long(slider_am_container_size))
- Logger.info("DBG: Done putting all configs")
+ self.logger.info("DBG: Done putting all configs")
#TODO: What is this doing? What error will be displayed on the UI if something like this is hit?
def recommendDefaultLlapConfiguration(self, configurations, services, hosts):
- Logger.info("DBG: Something likely went wrong. recommendDefaultLlapConfiguration")
+ self.logger.info("DBG: Something likely went wrong. recommendDefaultLlapConfiguration")
putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services)
putHiveInteractiveSitePropertyAttribute = self.putPropertyAttribute(configurations, self.HIVE_INTERACTIVE_SITE)
@@ -1344,7 +1343,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
elif hsi_env and 'num_llap_nodes' in hsi_env:
num_llap_nodes = hsi_env['num_llap_nodes']
else:
- Logger.error("Couldn't retrieve Hive Server 'num_llap_nodes' config. Setting value to {0}".format(num_llap_nodes))
+ self.logger.error("Couldn't retrieve Hive Server 'num_llap_nodes' config. Setting value to {0}".format(num_llap_nodes))
return float(num_llap_nodes)
@@ -1370,10 +1369,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
else:
calculated_hive_tez_container_size = 4096
- Logger.info("DBG: Calculated and returning 'hive_tez_container_size' : {0}".format(calculated_hive_tez_container_size))
+ self.logger.info("DBG: Calculated and returning 'hive_tez_container_size' : {0}".format(calculated_hive_tez_container_size))
return calculated_hive_tez_container_size
else:
- Logger.info("DBG: Returning 'hive_tez_container_size' : {0}".format(hive_tez_container_size))
+ self.logger.info("DBG: Returning 'hive_tez_container_size' : {0}".format(hive_tez_container_size))
return hive_tez_container_size
def get_hive_tez_container_size(self, services):
@@ -1406,16 +1405,16 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
# Check if 'llap_headroom_space' is modified in current SA invocation.
if 'hive-interactive-env' in configurations and 'llap_headroom_space' in configurations['hive-interactive-env']['properties']:
hive_container_size = float(configurations['hive-interactive-env']['properties']['llap_headroom_space'])
- Logger.info("'llap_headroom_space' read from configurations as : {0}".format(llap_headroom_space))
+ self.logger.info("'llap_headroom_space' read from configurations as : {0}".format(llap_headroom_space))
if llap_headroom_space is None:
# Check if 'llap_headroom_space' is input in services array.
if 'llap_headroom_space' in services['configurations']['hive-interactive-env']['properties']:
llap_headroom_space = float(services['configurations']['hive-interactive-env']['properties']['llap_headroom_space'])
- Logger.info("'llap_headroom_space' read from services as : {0}".format(llap_headroom_space))
+ self.logger.info("'llap_headroom_space' read from services as : {0}".format(llap_headroom_space))
if not llap_headroom_space or llap_headroom_space < 1:
llap_headroom_space = 6144 # 6GB
- Logger.info("Couldn't read 'llap_headroom_space' from services or configurations. Returing default value : 6144 bytes")
+ self.logger.info("Couldn't read 'llap_headroom_space' from services or configurations. Returing default value : 6144 bytes")
return llap_headroom_space
@@ -1444,15 +1443,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
# Check if services["changed-configurations"] is empty and 'yarn.scheduler.minimum-allocation-mb' is modified in current ST invocation.
if not services["changed-configurations"] and yarn_site and yarn_min_allocation_property in yarn_site:
yarn_min_container_size = yarn_site[yarn_min_allocation_property]
- Logger.info("DBG: 'yarn.scheduler.minimum-allocation-mb' read from output as : {0}".format(yarn_min_container_size))
+ self.logger.info("DBG: 'yarn.scheduler.minimum-allocation-mb' read from output as : {0}".format(yarn_min_container_size))
# Check if 'yarn.scheduler.minimum-allocation-mb' is input in services array.
elif yarn_site_properties and yarn_min_allocation_property in yarn_site_properties:
yarn_min_container_size = yarn_site_properties[yarn_min_allocation_property]
- Logger.info("DBG: 'yarn.scheduler.minimum-allocation-mb' read from services as : {0}".format(yarn_min_container_size))
+ self.logger.info("DBG: 'yarn.scheduler.minimum-allocation-mb' read from services as : {0}".format(yarn_min_container_size))
if not yarn_min_container_size:
- Logger.error("{0} was not found in the configuration".format(yarn_min_allocation_property))
+ self.logger.error("{0} was not found in the configuration".format(yarn_min_allocation_property))
return yarn_min_container_size
@@ -1482,10 +1481,10 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
elif total_cluster_capacity > 98304:
calculated_tez_am_resource_memory_mb = 4096
- Logger.info("DBG: Calculated and returning 'tez_am_resource_memory_mb' as : {0}".format(calculated_tez_am_resource_memory_mb))
+ self.logger.info("DBG: Calculated and returning 'tez_am_resource_memory_mb' as : {0}".format(calculated_tez_am_resource_memory_mb))
return float(calculated_tez_am_resource_memory_mb)
else:
- Logger.info("DBG: Returning 'tez_am_resource_memory_mb' as : {0}".format(tez_am_resource_memory_mb))
+ self.logger.info("DBG: Returning 'tez_am_resource_memory_mb' as : {0}".format(tez_am_resource_memory_mb))
return float(tez_am_resource_memory_mb)
def get_tez_am_resource_memory_mb(self, services):
@@ -1544,7 +1543,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
(2). Updates 'llap' queue capacity and state, if current selected queue is 'llap', and only 2 queues exist
at root level : 'default' and 'llap'.
"""
- Logger.info("Determining creation/adjustment of 'capacity-scheduler' for 'llap' queue.")
+ self.logger.info("Determining creation/adjustment of 'capacity-scheduler' for 'llap' queue.")
putHiveInteractiveEnvProperty = self.putProperty(configurations, "hive-interactive-env", services)
putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services)
putHiveInteractiveEnvPropertyAttribute = self.putPropertyAttribute(configurations, "hive-interactive-env")
@@ -1629,7 +1628,7 @@ yarn.scheduler.capacity.root.{0}.acl_administer_queue={2}
yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_queue_name, llap_queue_cap_perc, hive_user)
putCapSchedProperty("capacity-scheduler", updated_cap_sched_configs_str)
- Logger.info("Updated 'capacity-scheduler' configs as one concatenated string.")
+ self.logger.info("Updated 'capacity-scheduler' configs as one concatenated string.")
else:
# If capacity-scheduler configs are received as a dictionary (generally 1st time), we deposit the changed
# values back as dictionary itself.
@@ -1661,16 +1660,16 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu
putCapSchedProperty("yarn.scheduler.capacity.root." + llap_queue_name + ".acl_administer_queue", hive_user)
putCapSchedProperty("yarn.scheduler.capacity.root." + llap_queue_name + ".maximum-am-resource-percent", "1")
- Logger.info("Updated 'capacity-scheduler' configs as a dictionary.")
+ self.logger.info("Updated 'capacity-scheduler' configs as a dictionary.")
updated_cap_sched_configs_as_dict = True
if updated_cap_sched_configs_str or updated_cap_sched_configs_as_dict:
if len(leafQueueNames) == 1: # 'llap' queue didn't exist before
- Logger.info("Created YARN Queue : '{0}' with capacity : {1}%. Adjusted 'default' queue capacity to : {2}%" \
+ self.logger.info("Created YARN Queue : '{0}' with capacity : {1}%. Adjusted 'default' queue capacity to : {2}%" \
.format(llap_queue_name, llap_queue_cap_perc, adjusted_default_queue_cap))
else: # Queue existed, only adjustments done.
- Logger.info("Adjusted YARN Queue : '{0}'. Current capacity : {1}%. State: RUNNING.".format(llap_queue_name, llap_queue_cap_perc))
- Logger.info("Adjusted 'default' queue capacity to : {0}%".format(adjusted_default_queue_cap))
+ self.logger.info("Adjusted YARN Queue : '{0}'. Current capacity : {1}%. State: RUNNING.".format(llap_queue_name, llap_queue_cap_perc))
+ self.logger.info("Adjusted 'default' queue capacity to : {0}%".format(adjusted_default_queue_cap))
# Update Hive 'hive.llap.daemon.queue.name' prop to use 'llap' queue.
putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', llap_queue_name)
@@ -1678,9 +1677,9 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu
# Update 'hive.llap.daemon.queue.name' prop combo entries and llap capacity slider visibility.
self.setLlapDaemonQueuePropAttributes(services, configurations)
else:
- Logger.debug("Not creating/adjusting {0} queue. Current YARN queues : {1}".format(llap_queue_name, list(leafQueueNames)))
+ self.logger.debug("Not creating/adjusting {0} queue. Current YARN queues : {1}".format(llap_queue_name, list(leafQueueNames)))
else:
- Logger.error("Couldn't retrieve 'capacity-scheduler' properties while doing YARN queue adjustment for Hive Server Interactive.")
+ self.logger.error("Couldn't retrieve 'capacity-scheduler' properties while doing YARN queue adjustment for Hive Server Interactive.")
"""
Checks and sees (1). If only two leaf queues exist at root level, namely: 'default' and 'llap',
@@ -1706,7 +1705,7 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu
if 'yarn.scheduler.capacity.root.'+llap_queue_name+'.state' in capacity_scheduler_properties.keys():
currLlapQueueState = capacity_scheduler_properties.get('yarn.scheduler.capacity.root.'+llap_queue_name+'.state')
else:
- Logger.error("{0} queue 'state' property not present in capacity scheduler. Skipping adjusting queues.".format(llap_queue_name))
+ self.logger.error("{0} queue 'state' property not present in capacity scheduler. Skipping adjusting queues.".format(llap_queue_name))
return
if currLlapQueueState == 'RUNNING':
DEFAULT_MAX_CAPACITY = '100'
@@ -1739,28 +1738,28 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu
elif prop.startswith('yarn.'):
updated_llap_queue_configs = updated_llap_queue_configs + prop + "=" + val + "\n"
else:
- Logger.debug("{0} queue state is : {1}. Skipping adjusting queues.".format(llap_queue_name, currLlapQueueState))
+ self.logger.debug("{0} queue state is : {1}. Skipping adjusting queues.".format(llap_queue_name, currLlapQueueState))
return
if updated_default_queue_configs and updated_llap_queue_configs:
putCapSchedProperty("capacity-scheduler", updated_default_queue_configs+updated_llap_queue_configs)
- Logger.info("Changed YARN '{0}' queue state to 'STOPPED', and capacity to 0%. Adjusted 'default' queue capacity to : {1}%" \
+ self.logger.info("Changed YARN '{0}' queue state to 'STOPPED', and capacity to 0%. Adjusted 'default' queue capacity to : {1}%" \
.format(llap_queue_name, DEFAULT_MAX_CAPACITY))
# Update Hive 'hive.llap.daemon.queue.name' prop to use 'default' queue.
putHiveInteractiveSiteProperty('hive.llap.daemon.queue.name', self.YARN_ROOT_DEFAULT_QUEUE_NAME)
putHiveInteractiveSiteProperty('hive.server2.tez.default.queues', self.YARN_ROOT_DEFAULT_QUEUE_NAME)
else:
- Logger.debug("Not removing '{0}' queue as number of Queues not equal to 2. Current YARN queues : {1}".format(llap_queue_name, list(leafQueueNames)))
+ self.logger.debug("Not removing '{0}' queue as number of Queues not equal to 2. Current YARN queues : {1}".format(llap_queue_name, list(leafQueueNames)))
else:
- Logger.error("Couldn't retrieve 'capacity-scheduler' properties while doing YARN queue adjustment for Hive Server Interactive.")
+ self.logger.error("Couldn't retrieve 'capacity-scheduler' properties while doing YARN queue adjustment for Hive Server Interactive.")
def setLlapDaemonQueuePropAttributes(self, services, configurations):
"""
Checks and sets the 'Hive Server Interactive' 'hive.llap.daemon.queue.name' config Property Attributes. Takes into
account that 'capacity-scheduler' may have changed (got updated) in current Stack Advisor invocation.
"""
- Logger.info("Determining 'hive.llap.daemon.queue.name' config Property Attributes.")
+ self.logger.info("Determining 'hive.llap.daemon.queue.name' config Property Attributes.")
#TODO Determine if this is doing the right thing if some queue is setup with capacity=0, or is STOPPED. Maybe don't list it.
putHiveInteractiveSitePropertyAttribute = self.putPropertyAttribute(configurations, self.HIVE_INTERACTIVE_SITE)
@@ -1779,31 +1778,31 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu
for property in cap_sched_props_as_str:
key, sep, value = property.partition("=")
capacity_scheduler_properties[key] = value
- Logger.info("'capacity-scheduler' configs is set as a single '\\n' separated string in current invocation. "
+ self.logger.info("'capacity-scheduler' configs is set as a single '\\n' separated string in current invocation. "
"count(configurations['capacity-scheduler']['properties']['capacity-scheduler']) = "
"{0}".format(len(capacity_scheduler_properties)))
else:
- Logger.info("Read configurations['capacity-scheduler']['properties']['capacity-scheduler'] is : {0}".format(cap_sched_props_as_str))
+ self.logger.info("Read configurations['capacity-scheduler']['properties']['capacity-scheduler'] is : {0}".format(cap_sched_props_as_str))
else:
- Logger.info("configurations['capacity-scheduler']['properties']['capacity-scheduler'] : {0}.".format(cap_sched_props_as_str))
+ self.logger.info("configurations['capacity-scheduler']['properties']['capacity-scheduler'] : {0}.".format(cap_sched_props_as_str))
# if 'capacity_scheduler_properties' is empty, implies we may have 'capacity-scheduler' configs as dictionary
# in configurations, if 'capacity-scheduler' changed in current invocation.
if not capacity_scheduler_properties:
if isinstance(cap_sched_props_as_dict, dict) and len(cap_sched_props_as_dict) > 1:
capacity_scheduler_properties = cap_sched_props_as_dict
- Logger.info("'capacity-scheduler' changed in current Stack Advisor invocation. Retrieved the configs as dictionary from configurations.")
+ self.logger.info("'capacity-scheduler' changed in current Stack Advisor invocation. Retrieved the configs as dictionary from configurations.")
else:
- Logger.info("Read configurations['capacity-scheduler']['properties'] is : {0}".format(cap_sched_props_as_dict))
+ self.logger.info("Read configurations['capacity-scheduler']['properties'] is : {0}".format(cap_sched_props_as_dict))
else:
- Logger.info("'capacity-scheduler' not modified in the current Stack Advisor invocation.")
+ self.logger.info("'capacity-scheduler' not modified in the current Stack Advisor invocation.")
# if 'capacity_scheduler_properties' is still empty, implies 'capacity_scheduler' wasn't change in current
# SA invocation. Thus, read it from input : 'services'.
if not capacity_scheduler_properties:
capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
- Logger.info("'capacity-scheduler' not changed in current Stack Advisor invocation. Retrieved the configs from services.")
+ self.logger.info("'capacity-scheduler' not changed in current Stack Advisor invocation. Retrieved the configs from services.")
# Get set of current YARN leaf queues.
leafQueueNames = self.getAllYarnLeafQueues(capacity_scheduler_properties)
@@ -1811,9 +1810,9 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu
leafQueues = [{"label": str(queueName), "value": queueName} for queueName in leafQueueNames]
leafQueues = sorted(leafQueues, key=lambda q: q['value'])
putHiveInteractiveSitePropertyAttribute("hive.llap.daemon.queue.name", "entries", leafQueues)
- Logger.info("'hive.llap.daemon.queue.name' config Property Attributes set to : {0}".format(leafQueues))
+ self.logger.info("'hive.llap.daemon.queue.name' config Property Attributes set to : {0}".format(leafQueues))
else:
- Logger.error("Problem retrieving YARN queues. Skipping updating HIVE Server Interactve "
+ self.logger.error("Problem retrieving YARN queues. Skipping updating HIVE Server Interactve "
"'hive.server2.tez.default.queues' property attributes.")
def __getQueueCapacityKeyFromCapacityScheduler(self, capacity_scheduler_properties, llap_daemon_selected_queue_name):
@@ -1827,7 +1826,7 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu
for key in cap_sched_keys:
# Expected capacity prop key is of form : 'yarn.scheduler.capacity.<one or more queues in path separated by '.'>.[llap_daemon_selected_queue_name].capacity'
if key.endswith(llap_daemon_selected_queue_name+".capacity") and key.startswith("yarn.scheduler.capacity.root"):
- Logger.info("DBG: Selected queue name as: " + key)
+ self.logger.info("DBG: Selected queue name as: " + key)
llap_selected_queue_cap_key = key
break;
return llap_selected_queue_cap_key
@@ -1858,14 +1857,14 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu
for key in cap_sched_keys:
if key.endswith("."+llap_daemon_selected_queue_name+".maximum-am-resource-percent"):
llap_selected_queue_am_percent_key = key
- Logger.info("AM percent key got for '{0}' queue is : '{1}'".format(llap_daemon_selected_queue_name, llap_selected_queue_am_percent_key))
+ self.logger.info("AM percent key got for '{0}' queue is : '{1}'".format(llap_daemon_selected_queue_name, llap_selected_queue_am_percent_key))
break;
if llap_selected_queue_am_percent_key is None:
- Logger.info("Returning default AM percent value : '0.1' for queue : {0}".format(llap_daemon_selected_queue_name))
+ self.logger.info("Returning default AM percent value : '0.1' for queue : {0}".format(llap_daemon_selected_queue_name))
return 0.1 # Default value to use if we couldn't retrieve queue's corresponding AM Percent key.
else:
llap_selected_queue_am_percent = capacity_scheduler_properties.get(llap_selected_queue_am_percent_key)
- Logger.info("Returning read value for key '{0}' as : '{1}' for queue : '{2}'".format(llap_selected_queue_am_percent_key,
+ self.logger.info("Returning read value for key '{0}' as : '{1}' for queue : '{2}'".format(llap_selected_queue_am_percent_key,
llap_selected_queue_am_percent,
llap_daemon_selected_queue_name))
return llap_selected_queue_am_percent
@@ -1874,7 +1873,7 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu
"""
Calculates the total available capacity for the passed-in YARN queue of any level based on the percentages.
"""
- Logger.info("Entered __getSelectedQueueTotalCap fn() with llap_daemon_selected_queue_name= '{0}'.".format(llap_daemon_selected_queue_name))
+ self.logger.info("Entered __getSelectedQueueTotalCap fn() with llap_daemon_selected_queue_name= '{0}'.".format(llap_daemon_selected_queue_name))
available_capacity = total_cluster_capacity
queue_cap_key = self.__getQueueCapacityKeyFromCapacityScheduler(capacity_scheduler_properties, llap_daemon_selected_queue_name)
if queue_cap_key:
@@ -1884,13 +1883,13 @@ yarn.scheduler.capacity.root.{0}.maximum-am-resource-percent=1""".format(llap_qu
queue_path = queue_cap_key[24:] # Strip from beginning 'yarn.scheduler.capacity.'
queue_path = queue_path[0:-9] # Strip from end '.capacity'
queues_list = queue_path.split('.')
- Logger.info("Queue list : {0}".format(queues_list))
+ self.logger.info("Queue list : {0}".format(queues_list))
if queues_list:
for queue in queues_list:
queue_cap_key = self.__getQueueCapacityKeyFromCapacityScheduler(capacity_scheduler_properties, queue)
queue_cap_perc = float(capacity_scheduler_properties.get(queue_cap_key))
available_capacity = queue_cap_perc / 100 * available_capacity
- Logger.info("Total capacity available for queue {0} is : {1}".format(queue, available_capacity))
+ self.logger.info("Total capacity available for queue {0} is : {1}".format(queue, available_capacity))
# returns the capacity calculated for passed-in queue in 'llap_daemon_selected_queue_name'.
return available_capacity
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ef14f56/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 53ff007..7881917 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -16,7 +16,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
-from resource_management.core.logger import Logger
import json
import re
from resource_management.libraries.functions import format
@@ -25,7 +24,7 @@ from resource_management.libraries.functions import format
class HDP26StackAdvisor(HDP25StackAdvisor):
def __init__(self):
super(HDP26StackAdvisor, self).__init__()
- Logger.initialize_logger()
+ self.initialize_logger("HDP26StackAdvisor")
def getServiceConfigurationRecommenderDict(self):
parentRecommendConfDict = super(HDP26StackAdvisor, self).getServiceConfigurationRecommenderDict()
@@ -181,11 +180,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
putYarnSiteProperty('yarn.log.server.web-service.url',webservice_url )
if ranger_yarn_plugin_enabled and 'ranger-yarn-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-yarn-plugin-properties']['properties']:
- Logger.info("Setting Yarn Repo user for Ranger.")
+ self.logger.info("Setting Yarn Repo user for Ranger.")
putRangerYarnPluginProperty = self.putProperty(configurations, "ranger-yarn-plugin-properties", services)
putRangerYarnPluginProperty("REPOSITORY_CONFIG_USERNAME",yarn_user)
else:
- Logger.info("Not setting Yarn Repo user for Ranger.")
+ self.logger.info("Not setting Yarn Repo user for Ranger.")
def getMetadataConnectionString(self, database_type):
driverDict = {
@@ -282,7 +281,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
else:
webapp_address = services["configurations"]["yarn-site"]["properties"]["yarn.timeline-service.webapp.https.address"]
propertyValue = "https://"+webapp_address+"/ws/v1/applicationhistory"
- Logger.info("validateYarnSiteConfigurations: recommended value for webservice url"+services["configurations"]["yarn-site"]["properties"]["yarn.log.server.web-service.url"])
+ self.logger.info("validateYarnSiteConfigurations: recommended value for webservice url"+services["configurations"]["yarn-site"]["properties"]["yarn.log.server.web-service.url"])
if services["configurations"]["yarn-site"]["properties"]["yarn.log.server.web-service.url"] != propertyValue:
validationItems = [
{"config-name": "yarn.log.server.web-service.url",
@@ -325,7 +324,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
tez_jvm_updated_opts = tez_jvm_opts + jvmGCParams + "{{heap_dump_opts}}"
putTezProperty('tez.am.launch.cmd-opts', tez_jvm_updated_opts)
putTezProperty('tez.task.launch.cmd-opts', tez_jvm_updated_opts)
- Logger.info("Updated 'tez-site' config 'tez.task.launch.cmd-opts' and 'tez.am.launch.cmd-opts' as "
+ self.logger.info("Updated 'tez-site' config 'tez.task.launch.cmd-opts' and 'tez.am.launch.cmd-opts' as "
": {0}".format(tez_jvm_updated_opts))
def recommendRangerConfigurations(self, configurations, clusterData, services, hosts):
@@ -390,11 +389,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
ranger_hdfs_plugin_enabled = False
if ranger_hdfs_plugin_enabled and 'ranger-hdfs-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-hdfs-plugin-properties']['properties']:
- Logger.info("Setting HDFS Repo user for Ranger.")
+ self.logger.info("Setting HDFS Repo user for Ranger.")
putRangerHDFSPluginProperty = self.putProperty(configurations, "ranger-hdfs-plugin-properties", services)
putRangerHDFSPluginProperty("REPOSITORY_CONFIG_USERNAME",hdfs_user)
else:
- Logger.info("Not setting HDFS Repo user for Ranger.")
+ self.logger.info("Not setting HDFS Repo user for Ranger.")
def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts):
super(HDP26StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData, services, hosts)
@@ -411,11 +410,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
ranger_hive_plugin_enabled = False
if ranger_hive_plugin_enabled and 'ranger-hive-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-hive-plugin-properties']['properties']:
- Logger.info("Setting Hive Repo user for Ranger.")
+ self.logger.info("Setting Hive Repo user for Ranger.")
putRangerHivePluginProperty = self.putProperty(configurations, "ranger-hive-plugin-properties", services)
putRangerHivePluginProperty("REPOSITORY_CONFIG_USERNAME",hive_user)
else:
- Logger.info("Not setting Hive Repo user for Ranger.")
+ self.logger.info("Not setting Hive Repo user for Ranger.")
def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
super(HDP26StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts)
@@ -432,11 +431,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
ranger_hbase_plugin_enabled = False
if ranger_hbase_plugin_enabled and 'ranger-hbase-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-hbase-plugin-properties']['properties']:
- Logger.info("Setting Hbase Repo user for Ranger.")
+ self.logger.info("Setting Hbase Repo user for Ranger.")
putRangerHbasePluginProperty = self.putProperty(configurations, "ranger-hbase-plugin-properties", services)
putRangerHbasePluginProperty("REPOSITORY_CONFIG_USERNAME",hbase_user)
else:
- Logger.info("Not setting Hbase Repo user for Ranger.")
+ self.logger.info("Not setting Hbase Repo user for Ranger.")
def recommendKAFKAConfigurations(self, configurations, clusterData, services, hosts):
super(HDP26StackAdvisor, self).recommendKAFKAConfigurations(configurations, clusterData, services, hosts)
@@ -453,8 +452,8 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
ranger_kafka_plugin_enabled = False
if ranger_kafka_plugin_enabled and 'ranger-kafka-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-kafka-plugin-properties']['properties']:
- Logger.info("Setting Kafka Repo user for Ranger.")
+ self.logger.info("Setting Kafka Repo user for Ranger.")
putRangerKafkaPluginProperty = self.putProperty(configurations, "ranger-kafka-plugin-properties", services)
putRangerKafkaPluginProperty("REPOSITORY_CONFIG_USERNAME",kafka_user)
else:
- Logger.info("Not setting Kafka Repo user for Ranger.")
+ self.logger.info("Not setting Kafka Repo user for Ranger.")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ef14f56/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
index c7d9327..c1dcce8 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/stack_advisor.py
@@ -24,7 +24,6 @@ import os
from math import ceil
# Local Imports
-from resource_management.core.logger import Logger
from stack_advisor import DefaultStackAdvisor
class HDPWIN21StackAdvisor(DefaultStackAdvisor):
@@ -32,6 +31,8 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
def __init__(self):
super(HDPWIN21StackAdvisor, self).__init__()
+ self.initialize_logger("HDPWIN21StackAdvisor")
+
self.modifyMastersWithMultipleInstances()
self.modifyCardinalitiesDict()
self.modifyHeapSizeProperties()
@@ -409,9 +410,9 @@ class HDPWIN21StackAdvisor(DefaultStackAdvisor):
siteProperties = getSiteProperties(configurations, siteName)
if siteProperties is not None:
siteRecommendations = recommendedDefaults[siteName]["properties"]
- print("SiteName: %s, method: %s\n" % (siteName, method.__name__))
- print("Site properties: %s\n" % str(siteProperties))
- print("Recommendations: %s\n********\n" % str(siteRecommendations))
+ self.logger.info("SiteName: %s, method: %s\n" % (siteName, method.__name__))
+ self.logger.info("Site properties: %s\n" % str(siteProperties))
+ self.logger.info("Recommendations: %s\n********\n" % str(siteRecommendations))
resultItems = method(siteProperties, siteRecommendations, configurations, services, hosts)
items.extend(resultItems)
clusterWideItems = self.validateClusterConfigurations(configurations, services, hosts)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ef14f56/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
index a3d9598..7d11640 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py
@@ -25,7 +25,6 @@ import sys
from urlparse import urlparse
# Local Imports
-from resource_management.core.logger import Logger
def getSiteProperties(configurations, siteName):
siteConfig = configurations.get(siteName)
@@ -37,7 +36,7 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor):
def __init__(self):
super(HDPWIN22StackAdvisor, self).__init__()
- Logger.initialize_logger()
+ self.initialize_logger("HDPWIN22StackAdvisor")
self.modifyMastersWithMultipleInstances()
self.modifyCardinalitiesDict()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0ef14f56/ambari-server/src/main/resources/stacks/PERF/1.0/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/PERF/1.0/services/stack_advisor.py
index 43d417e..835311a 100644
--- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/stack_advisor.py
@@ -20,7 +20,6 @@ limitations under the License.
# Python Imports
# Local Imports
-from resource_management.core.logger import Logger
from stack_advisor import DefaultStackAdvisor
@@ -28,7 +27,7 @@ class PERF10StackAdvisor(DefaultStackAdvisor):
def __init__(self):
super(PERF10StackAdvisor, self).__init__()
- Logger.initialize_logger()
+ self.initialize_logger("PERF10StackAdvisor")
def getServiceConfigurationRecommenderDict(self):
return {}