You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2017/03/30 12:49:11 UTC
[4/4] ambari git commit: AMBARI-20617. Display log level,
method name from stack_advisor in ambari-server.log (dlysnichenko)
AMBARI-20617. Display log level, method name from stack_advisor in ambari-server.log (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9feb5d0f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9feb5d0f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9feb5d0f
Branch: refs/heads/trunk
Commit: 9feb5d0f9a4d2d287d6724896668fc297aa0a778
Parents: fb8d463
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Thu Mar 30 15:48:24 2017 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Thu Mar 30 15:48:24 2017 +0300
----------------------------------------------------------------------
.../HDFS/3.0.0.3.0/service_advisor.py | 27 +-
.../HIVE/2.1.0.3.0/service_advisor.py | 25 +-
.../SLIDER/0.91.0.3.0/service_advisor.py | 7 +-
.../SPARK/2.2.0/service_advisor.py | 7 +-
.../TEZ/0.9.0.3.0/service_advisor.py | 9 +-
.../YARN/3.0.0.3.0/service_advisor.py | 259 ++++++++---------
.../ZOOKEEPER/3.4.9/service_advisor.py | 13 +-
.../stacks/BIGTOP/0.8/services/stack_advisor.py | 3 +-
.../stacks/HDP/2.0.6/services/stack_advisor.py | 7 +-
.../stacks/HDP/2.1/services/stack_advisor.py | 3 +-
.../stacks/HDP/2.2/services/stack_advisor.py | 3 +-
.../stacks/HDP/2.3/services/stack_advisor.py | 3 +-
.../stacks/HDP/2.5/services/stack_advisor.py | 277 +++++++++----------
.../stacks/HDP/2.6/services/stack_advisor.py | 27 +-
.../stacks/HDPWIN/2.1/services/stack_advisor.py | 9 +-
.../stacks/HDPWIN/2.2/services/stack_advisor.py | 3 +-
.../stacks/PERF/1.0/services/stack_advisor.py | 3 +-
.../src/main/resources/stacks/stack_advisor.py | 88 +++---
.../stacks/ODPi/2.0/services/stack_advisor.py | 21 +-
19 files changed, 404 insertions(+), 390 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9feb5d0f/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/service_advisor.py
index a37ebc67..356ad59 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/service_advisor.py
@@ -24,7 +24,6 @@ import traceback
import inspect
# Local imports
-from resource_management.core.logger import Logger
from resource_management.libraries.functions.mounted_dirs_helper import get_mounts_with_multiple_data_dirs
@@ -46,6 +45,8 @@ class HDFSServiceAdvisor(service_advisor.ServiceAdvisor):
self.as_super = super(HDFSServiceAdvisor, self)
self.as_super.__init__(*args, **kwargs)
+ self.initialize_logger("HDFSServiceAdvisor")
+
# Always call these methods
self.modifyMastersWithMultipleInstances()
self.modifyCardinalitiesDict()
@@ -120,7 +121,7 @@ class HDFSServiceAdvisor(service_advisor.ServiceAdvisor):
Get a list of errors.
Must be overriden in child class.
"""
- Logger.info("Class: %s, Method: %s. Validating Service Component Layout." %
+ self.logger.info("Class: %s, Method: %s. Validating Service Component Layout." %
(self.__class__.__name__, inspect.stack()[0][3]))
# HDFS allows NameNode and Secondary NameNode to be on the same host.
@@ -131,7 +132,7 @@ class HDFSServiceAdvisor(service_advisor.ServiceAdvisor):
Entry point.
Must be overriden in child class.
"""
- Logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
+ self.logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
(self.__class__.__name__, inspect.stack()[0][3]))
# Due to the existing stack inheritance, make it clear where each calculation came from.
@@ -146,7 +147,7 @@ class HDFSServiceAdvisor(service_advisor.ServiceAdvisor):
Validate configurations for the service. Return a list of errors.
The code for this function should be the same for each Service Advisor.
"""
- Logger.info("Class: %s, Method: %s. Validating Configurations." %
+ self.logger.info("Class: %s, Method: %s. Validating Configurations." %
(self.__class__.__name__, inspect.stack()[0][3]))
validator = HDFSValidator()
@@ -167,7 +168,7 @@ class HDFSRecommender(service_advisor.ServiceAdvisor):
"""
Recommend configurations for this service based on HDP 2.0.6.
"""
- Logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
+ self.logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
(self.__class__.__name__, inspect.stack()[0][3]))
putHDFSProperty = self.putProperty(configurations, "hadoop-env", services)
@@ -175,7 +176,7 @@ class HDFSRecommender(service_advisor.ServiceAdvisor):
putHDFSSitePropertyAttributes = self.putPropertyAttribute(configurations, "hdfs-site")
totalAvailableRam = clusterData['totalAvailableRam']
- Logger.info("Class: %s, Method: %s. Total Available Ram: %s" % (self.__class__.__name__, inspect.stack()[0][3], str(totalAvailableRam)))
+ self.logger.info("Class: %s, Method: %s. Total Available Ram: %s" % (self.__class__.__name__, inspect.stack()[0][3], str(totalAvailableRam)))
putHDFSProperty('namenode_heapsize', max(int(totalAvailableRam / 2), 1024))
putHDFSProperty = self.putProperty(configurations, "hadoop-env", services)
putHDFSProperty('namenode_opt_newsize', max(int(totalAvailableRam / 8), 128))
@@ -194,7 +195,7 @@ class HDFSRecommender(service_advisor.ServiceAdvisor):
if len(namenodes.split(',')) > 1:
putHDFSSitePropertyAttributes("dfs.namenode.rpc-address", "delete", "true")
- Logger.info("Class: %s, Method: %s. HDFS nameservices: %s" %
+ self.logger.info("Class: %s, Method: %s. HDFS nameservices: %s" %
(self.__class__.__name__, inspect.stack()[0][3], str(nameServices)))
hdfs_mount_properties = [
@@ -203,7 +204,7 @@ class HDFSRecommender(service_advisor.ServiceAdvisor):
("dfs.namenode.checkpoint.dir", "SECONDARY_NAMENODE", "/hadoop/hdfs/namesecondary", "single")
]
- Logger.info("Class: %s, Method: %s. Updating HDFS mount properties." %
+ self.logger.info("Class: %s, Method: %s. Updating HDFS mount properties." %
(self.__class__.__name__, inspect.stack()[0][3]))
self.updateMountProperties("hdfs-site", hdfs_mount_properties, configurations, services, hosts)
@@ -217,7 +218,7 @@ class HDFSRecommender(service_advisor.ServiceAdvisor):
hdfsSiteProperties["dfs.datanode.data.dir"] is not None:
dataDirs = hdfsSiteProperties["dfs.datanode.data.dir"].split(",")
- Logger.info("Class: %s, Method: %s. HDFS Data Dirs: %s" %
+ self.logger.info("Class: %s, Method: %s. HDFS Data Dirs: %s" %
(self.__class__.__name__, inspect.stack()[0][3], str(dataDirs)))
# dfs.datanode.du.reserved should be set to 10-15% of volume size
@@ -242,7 +243,7 @@ class HDFSRecommender(service_advisor.ServiceAdvisor):
if (not reservedSizeRecommendation) or (maxFreeVolumeSizeForHost and maxFreeVolumeSizeForHost < reservedSizeRecommendation):
reservedSizeRecommendation = maxFreeVolumeSizeForHost
- Logger.info("Class: %s, Method: %s. HDFS Datanode recommended reserved size: %d" %
+ self.logger.info("Class: %s, Method: %s. HDFS Datanode recommended reserved size: %d" %
(self.__class__.__name__, inspect.stack()[0][3], reservedSizeRecommendation))
if reservedSizeRecommendation:
@@ -290,11 +291,11 @@ class HDFSRecommender(service_advisor.ServiceAdvisor):
ranger_hdfs_plugin_enabled = False
if ranger_hdfs_plugin_enabled and 'ranger-hdfs-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-hdfs-plugin-properties']['properties']:
- Logger.info("Setting HDFS Repo user for Ranger.")
+ self.logger.info("Setting HDFS Repo user for Ranger.")
putRangerHDFSPluginProperty = self.putProperty(configurations, "ranger-hdfs-plugin-properties", services)
putRangerHDFSPluginProperty("REPOSITORY_CONFIG_USERNAME",hdfs_user)
else:
- Logger.info("Not setting HDFS Repo user for Ranger.")
+ self.logger.info("Not setting HDFS Repo user for Ranger.")
class HDFSValidator(service_advisor.ServiceAdvisor):
@@ -590,7 +591,7 @@ class HDFSValidator(service_advisor.ServiceAdvisor):
If Ranger service is present and the ranger plugin is enabled, check that the provider class is correctly set.
:return: A list of configuration validation problems.
"""
- Logger.info("Class: %s, Method: %s. Checking if Ranger service is present and if the provider class is using the Ranger Authorizer." %
+ self.logger.info("Class: %s, Method: %s. Checking if Ranger service is present and if the provider class is using the Ranger Authorizer." %
(self.__class__.__name__, inspect.stack()[0][3]))
# We can not access property hadoop.security.authentication from the
# other config (core-site). That's why we are using another heuristics here
http://git-wip-us.apache.org/repos/asf/ambari/blob/9feb5d0f/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/service_advisor.py
index c60339b..ef3e340 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/service_advisor.py
@@ -26,7 +26,6 @@ import socket
import fnmatch
-from resource_management.core.logger import Logger
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
@@ -45,6 +44,8 @@ class HiveServiceAdvisor(service_advisor.ServiceAdvisor):
self.as_super = super(HiveServiceAdvisor, self)
self.as_super.__init__(*args, **kwargs)
+ self.initialize_logger("HiveServiceAdvisor")
+
# Always call these methods
self.modifyMastersWithMultipleInstances()
self.modifyCardinalitiesDict()
@@ -118,7 +119,7 @@ class HiveServiceAdvisor(service_advisor.ServiceAdvisor):
Entry point.
Must be overriden in child class.
"""
- Logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
+ self.logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
(self.__class__.__name__, inspect.stack()[0][3]))
recommender = HiveRecommender()
@@ -136,7 +137,7 @@ class HiveServiceAdvisor(service_advisor.ServiceAdvisor):
Validate configurations for the service. Return a list of errors.
The code for this function should be the same for each Service Advisor.
"""
- Logger.info("Class: %s, Method: %s. Validating Configurations." %
+ self.logger.info("Class: %s, Method: %s. Validating Configurations." %
(self.__class__.__name__, inspect.stack()[0][3]))
validator = HiveValidator()
@@ -617,7 +618,7 @@ class HiveRecommender(service_advisor.ServiceAdvisor):
def recommendHIVEConfigurationsFromHDP25(self, configurations, clusterData, services, hosts):
- Logger.info("DBG: Invoked recommendHiveConfiguration")
+ self.logger.info("DBG: Invoked recommendHiveConfiguration")
putHiveInteractiveEnvProperty = self.putProperty(configurations, "hive-interactive-env", services)
putHiveInteractiveSiteProperty = self.putProperty(configurations, self.HIVE_INTERACTIVE_SITE, services)
@@ -642,9 +643,9 @@ class HiveRecommender(service_advisor.ServiceAdvisor):
if hive_tez_default_queue:
putHiveInteractiveSiteProperty("hive.server2.tez.default.queues", hive_tez_default_queue)
- Logger.debug("Updated 'hive.server2.tez.default.queues' config : '{0}'".format(hive_tez_default_queue))
+ self.logger.debug("Updated 'hive.server2.tez.default.queues' config : '{0}'".format(hive_tez_default_queue))
else:
- Logger.info("DBG: Setting 'num_llap_nodes' config's READ ONLY attribute as 'True'.")
+ self.logger.info("DBG: Setting 'num_llap_nodes' config's READ ONLY attribute as 'True'.")
putHiveInteractiveEnvProperty('enable_hive_interactive', 'false')
putHiveInteractiveEnvPropertyAttribute("num_llap_nodes", "read_only", "true")
@@ -669,11 +670,11 @@ class HiveRecommender(service_advisor.ServiceAdvisor):
ranger_hive_plugin_enabled = False
if ranger_hive_plugin_enabled and 'ranger-hive-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-hive-plugin-properties']['properties']:
- Logger.info("Setting Hive Repo user for Ranger.")
+ self.logger.info("Setting Hive Repo user for Ranger.")
putRangerHivePluginProperty = self.putProperty(configurations, "ranger-hive-plugin-properties", services)
putRangerHivePluginProperty("REPOSITORY_CONFIG_USERNAME",hive_user)
else:
- Logger.info("Not setting Hive Repo user for Ranger.")
+ self.logger.info("Not setting Hive Repo user for Ranger.")
def getDBDriver(self, databaseType):
@@ -1051,7 +1052,7 @@ class HiveValidator(service_advisor.ServiceAdvisor):
capacity_scheduler_properties, received_as_key_value_pair = self.getCapacitySchedulerProperties(services)
if not capacity_scheduler_properties:
- Logger.warning("Couldn't retrieve 'capacity-scheduler' properties while doing validation checks for Hive Server Interactive.")
+ self.logger.warning("Couldn't retrieve 'capacity-scheduler' properties while doing validation checks for Hive Server Interactive.")
return []
if hsi_site:
@@ -1070,7 +1071,7 @@ class HiveValidator(service_advisor.ServiceAdvisor):
"app to run".format(llap_queue_name, llap_queue_cap_perc, min_reqd_queue_cap_perc)
validationItems.append({"config-name": "hive.llap.daemon.queue.name", "item": self.getErrorItem(errMsg1)})
else:
- Logger.error("Couldn't retrieve '{0}' queue's capacity from 'capacity-scheduler' while doing validation checks for "
+ self.logger.error("Couldn't retrieve '{0}' queue's capacity from 'capacity-scheduler' while doing validation checks for "
"Hive Server Interactive.".format(llap_queue_name))
# Validate that current selected queue in 'hive.llap.daemon.queue.name' state is not STOPPED.
@@ -1081,10 +1082,10 @@ class HiveValidator(service_advisor.ServiceAdvisor):
.format(llap_queue_name, llap_selected_queue_state)
validationItems.append({"config-name": "hive.llap.daemon.queue.name","item": self.getErrorItem(errMsg2)})
else:
- Logger.error("Couldn't retrieve '{0}' queue's state from 'capacity-scheduler' while doing validation checks for "
+ self.logger.error("Couldn't retrieve '{0}' queue's state from 'capacity-scheduler' while doing validation checks for "
"Hive Server Interactive.".format(llap_queue_name))
else:
- Logger.error("Couldn't retrieve 'hive.llap.daemon.queue.name' config from 'hive-interactive-site' while doing "
+ self.logger.error("Couldn't retrieve 'hive.llap.daemon.queue.name' config from 'hive-interactive-site' while doing "
"validation checks for Hive Server Interactive.")
# Validate that 'hive.server2.enable.doAs' config is not set to 'true' for Hive2.
http://git-wip-us.apache.org/repos/asf/ambari/blob/9feb5d0f/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/service_advisor.py
index 6122f14..5614868 100644
--- a/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/SLIDER/0.91.0.3.0/service_advisor.py
@@ -23,7 +23,6 @@ import os
import traceback
-from resource_management.core.logger import Logger
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
@@ -43,6 +42,8 @@ class SliderServiceAdvisor(service_advisor.ServiceAdvisor):
self.as_super = super(SliderServiceAdvisor, self)
self.as_super.__init__(*args, **kwargs)
+ self.initialize_logger("SliderServiceAdvisor")
+
# Always call these methods
self.modifyMastersWithMultipleInstances()
self.modifyCardinalitiesDict()
@@ -113,7 +114,7 @@ class SliderServiceAdvisor(service_advisor.ServiceAdvisor):
Entry point.
Must be overriden in child class.
"""
- Logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
+ self.logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
(self.__class__.__name__, inspect.stack()[0][3]))
# Nothing to do
@@ -125,7 +126,7 @@ class SliderServiceAdvisor(service_advisor.ServiceAdvisor):
Validate configurations for the service. Return a list of errors.
The code for this function should be the same for each Service Advisor.
"""
- Logger.info("Class: %s, Method: %s. Validating Configurations." %
+ self.logger.info("Class: %s, Method: %s. Validating Configurations." %
(self.__class__.__name__, inspect.stack()[0][3]))
# Nothing to do
http://git-wip-us.apache.org/repos/asf/ambari/blob/9feb5d0f/ambari-server/src/main/resources/common-services/SPARK/2.2.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/service_advisor.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/service_advisor.py
index ee803b0..b876cd7 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/service_advisor.py
@@ -23,7 +23,6 @@ import os
import traceback
-from resource_management.core.logger import Logger
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
@@ -42,6 +41,8 @@ class SparkServiceAdvisor(service_advisor.ServiceAdvisor):
self.as_super = super(SparkServiceAdvisor, self)
self.as_super.__init__(*args, **kwargs)
+ self.initialize_logger("SparkServiceAdvisor")
+
# Always call these methods
self.modifyMastersWithMultipleInstances()
self.modifyCardinalitiesDict()
@@ -120,7 +121,7 @@ class SparkServiceAdvisor(service_advisor.ServiceAdvisor):
Entry point.
Must be overriden in child class.
"""
- Logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
+ self.logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
(self.__class__.__name__, inspect.stack()[0][3]))
recommender = SparkRecommender()
@@ -134,7 +135,7 @@ class SparkServiceAdvisor(service_advisor.ServiceAdvisor):
Validate configurations for the service. Return a list of errors.
The code for this function should be the same for each Service Advisor.
"""
- Logger.info("Class: %s, Method: %s. Validating Configurations." %
+ self.logger.info("Class: %s, Method: %s. Validating Configurations." %
(self.__class__.__name__, inspect.stack()[0][3]))
validator = SparkValidator()
http://git-wip-us.apache.org/repos/asf/ambari/blob/9feb5d0f/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/service_advisor.py
index cc84cc4..d71983f 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.9.0.3.0/service_advisor.py
@@ -26,7 +26,6 @@ import socket
import fnmatch
-from resource_management.core.logger import Logger
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
@@ -45,6 +44,8 @@ class TezServiceAdvisor(service_advisor.ServiceAdvisor):
self.as_super = super(TezServiceAdvisor, self)
self.as_super.__init__(*args, **kwargs)
+ self.initialize_logger("TezServiceAdvisor")
+
# Always call these methods
self.modifyMastersWithMultipleInstances()
self.modifyCardinalitiesDict()
@@ -115,7 +116,7 @@ class TezServiceAdvisor(service_advisor.ServiceAdvisor):
Entry point.
Must be overriden in child class.
"""
- Logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
+ self.logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
(self.__class__.__name__, inspect.stack()[0][3]))
recommender = TezRecommender()
@@ -131,7 +132,7 @@ class TezServiceAdvisor(service_advisor.ServiceAdvisor):
Validate configurations for the service. Return a list of errors.
The code for this function should be the same for each Service Advisor.
"""
- Logger.info("Class: %s, Method: %s. Validating Configurations." %
+ self.logger.info("Class: %s, Method: %s. Validating Configurations." %
(self.__class__.__name__, inspect.stack()[0][3]))
validator = TezValidator()
@@ -315,7 +316,7 @@ class TezRecommender(service_advisor.ServiceAdvisor):
tez_jvm_updated_opts = tez_jvm_opts + jvmGCParams + "{{heap_dump_opts}}"
putTezProperty('tez.am.launch.cmd-opts', tez_jvm_updated_opts)
putTezProperty('tez.task.launch.cmd-opts', tez_jvm_updated_opts)
- Logger.info("Updated 'tez-site' config 'tez.task.launch.cmd-opts' and 'tez.am.launch.cmd-opts' as "
+ self.logger.info("Updated 'tez-site' config 'tez.task.launch.cmd-opts' and 'tez.am.launch.cmd-opts' as "
": {0}".format(tez_jvm_updated_opts))