You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/04/28 17:56:42 UTC
ambari git commit: AMBARI-16158. takeover_config_merge.py should
process known template config files (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk 84a2595eb -> e1069651d
AMBARI-16158. takeover_config_merge.py should process known template config files (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e1069651
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e1069651
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e1069651
Branch: refs/heads/trunk
Commit: e1069651db0f503f15c8b31ec50f1932d65ab7be
Parents: 84a2595
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Apr 28 18:56:31 2016 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Apr 28 18:56:31 2016 +0300
----------------------------------------------------------------------
.../resources/scripts/takeover_config_merge.py | 160 +++++++++++++++----
1 file changed, 128 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e1069651/ambari-server/src/main/resources/scripts/takeover_config_merge.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/takeover_config_merge.py b/ambari-server/src/main/resources/scripts/takeover_config_merge.py
index ed1da07..8f11db4 100644
--- a/ambari-server/src/main/resources/scripts/takeover_config_merge.py
+++ b/ambari-server/src/main/resources/scripts/takeover_config_merge.py
@@ -23,43 +23,67 @@ import os
import logging
import tempfile
import json
+import re
import base64
import time
import xml
import xml.etree.ElementTree as ET
+import yaml
+import StringIO
+import ConfigParser
logger = logging.getLogger('AmbariTakeoverConfigMerge')
+LOG4J_HELP_TEXT = """
+JSON file should content map with {regex_path : <service>-log4j}
+Example:
+{".+/hadoop/.+/log4j.properties" : "hdfs-log4j",
+".+/etc/zookeeper/conf/log4j.properties" : "zookeeper-log4j"
+"c6401.ambari.apache.org/etc/hive/conf/log4j.properties" : "hive-log4j"}
+"""
-class ConfigMerge:
- INPUT_DIR = '/etc/hadoop'
- OUTPUT_DIR = '/tmp'
- OUT_FILENAME = 'ambari_takeover_config_merge.out'
- JSON_FILENAME = 'ambari_takeover_config_merge.json'
- SUPPORTED_EXTENSIONS = ['.xml']
- SUPPORTED_FILENAME_ENDINGS = ['-site']
+class Parser:
+ pass
- config_files_map = {}
+class YamlParser(Parser): # Used Yaml parser to read data into a map
+ def read_data_to_map(self, path):
+ configurations = {}
+ with open(path, 'r') as file:
+ try:
+ for name, value in yaml.load(file).iteritems():
+ if name != None:
+ configurations[name] = str(value)
+ except yaml.YAMLError:
+ logger.exception("Yaml parser error: ")
+ return None
+ return configurations
- def __init__(self, config_files_map):
- self.config_files_map = config_files_map
- @staticmethod
- def get_all_supported_files_grouped_by_name(extensions=SUPPORTED_EXTENSIONS):
- filePaths = {}
- for dirName, subdirList, fileList in os.walk(ConfigMerge.INPUT_DIR, followlinks=True):
- for file in fileList:
- root, ext = os.path.splitext(file)
- for filename_ending in ConfigMerge.SUPPORTED_FILENAME_ENDINGS:
- if root.endswith(filename_ending) and ext in extensions:
- if not file in filePaths:
- filePaths[file] = []
- filePaths[file].append(os.path.join(dirName, file))
+class PropertiesParser(Parser): # Used ConfigParser parser to read data into a map
+ def read_data_to_map(self, path):
+ configurations = {}
+ try :
+ #Adding dummy section to properties file content for use ConfigParser
+ properties_file_content = StringIO.StringIO()
+ properties_file_content.write('[dummysection]\n')
+ properties_file_content.write(open(path).read())
+ properties_file_content.seek(0, os.SEEK_SET)
+
+ cp = ConfigParser.ConfigParser()
+ cp.readfp(properties_file_content)
+
+ for section in cp._sections:
+ for name, value in cp._sections[section].iteritems():
+ if name != None:
+ configurations[name] = value
+ del configurations['__name__']
+ except:
+ logger.exception("ConfigParser error: ")
+ return configurations
- return filePaths
- # Used DOM parser to read data into a map
- def read_xml_data_to_map(self, path):
+class XmlParser(Parser): # Used DOM parser to read data into a map
+ def read_data_to_map(self, path):
configurations = {}
tree = ET.parse(path)
root = tree.getroot()
@@ -83,6 +107,65 @@ class ConfigMerge:
logger.debug("Following configurations found in {0}:\n{1}".format(path, configurations))
return configurations
+class ConfigMerge:
+
+ CONTENT_UNKNOWN_FILES_MAPPING_FILE = {}
+ INPUT_DIR = '/etc/hadoop'
+ OUTPUT_DIR = '/tmp'
+ OUT_FILENAME = 'ambari_takeover_config_merge.out'
+ JSON_FILENAME = 'ambari_takeover_config_merge.json'
+ PARSER_BY_EXTENSIONS = {'.xml' : XmlParser(), '.yaml' : YamlParser(), '.properties' : PropertiesParser()}
+ SUPPORTED_EXTENSIONS = ['.xml', '.yaml', '.properties']
+ UNKNOWN_FILES_MAPPING_FILE = None
+ SERVICE_TO_AMBARI_CONFIG_NAME = {
+ "storm.yaml": "storm-site"
+ }
+
+ NOT_MAPPED_FILES = ['log4j.properties']
+
+
+ config_files_map = {}
+
+ def __init__(self, config_files_map):
+ self.config_files_map = config_files_map
+
+ @staticmethod
+ def get_all_supported_files_grouped_by_name(extensions=SUPPORTED_EXTENSIONS):
+ filePaths = {}
+ for dirName, subdirList, fileList in os.walk(ConfigMerge.INPUT_DIR, followlinks=True):
+ for file in fileList:
+ root, ext = os.path.splitext(file)
+ if ext in extensions:
+ file_path = os.path.join(dirName, file)
+
+ config_name = None
+ if file in ConfigMerge.SERVICE_TO_AMBARI_CONFIG_NAME:
+ config_name = ConfigMerge.SERVICE_TO_AMBARI_CONFIG_NAME[file]
+
+ #hack for log4j.properties files
+ elif ConfigMerge.UNKNOWN_FILES_MAPPING_FILE:
+ for path_regex, name in ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE.iteritems():
+ match = re.match(path_regex, os.path.relpath(file_path, ConfigMerge.INPUT_DIR))
+ if match:
+ config_name = name
+ break
+
+ if not config_name:
+ if file in ConfigMerge.NOT_MAPPED_FILES:
+ if ConfigMerge.UNKNOWN_FILES_MAPPING_FILE:
+ logger.error("File {0} doesn't match any regex from {1}".format(file_path, ConfigMerge.UNKNOWN_FILES_MAPPING_FILE))
+ else:
+ logger.error("Cannot map {0} to Ambari config type. Please use -u option to specify config mapping for this file. \n"
+ "For more information use --help option for script".format(file_path))
+ continue
+ else:
+ config_name = file
+
+ if not config_name in filePaths:
+ filePaths[config_name] = []
+ filePaths[config_name].append((file_path, ConfigMerge.PARSER_BY_EXTENSIONS[ext]))
+ return filePaths
+
@staticmethod
def merge_configurations(filepath_to_configurations):
configuration_information_dict = {}
@@ -131,12 +214,15 @@ class ConfigMerge:
def perform_merge(self):
result_configurations = {}
has_conflicts = False
- for filename, paths in self.config_files_map.iteritems():
+ for filename, paths_and_parsers in self.config_files_map.iteritems():
filepath_to_configurations = {}
configuration_type = os.path.splitext(filename)[0]
- for path in paths:
- logger.debug("Read xml data from {0}".format(path))
- filepath_to_configurations[path] = self.read_xml_data_to_map(path)
+ for path_and_parser in paths_and_parsers:
+ path, parser = path_and_parser
+ logger.debug("Read data from {0}".format(path))
+ parsed_configurations_from_path = parser.read_data_to_map(path)
+ if parsed_configurations_from_path != None:
+ filepath_to_configurations[path] = parsed_configurations_from_path
merged_configurations, property_name_to_value_to_filepaths = ConfigMerge.merge_configurations(
filepath_to_configurations)
@@ -164,7 +250,6 @@ class ConfigMerge:
logger.info("Script successfully finished")
return 0
-
def main():
tempDir = tempfile.gettempdir()
outputDir = os.path.join(tempDir)
@@ -177,9 +262,8 @@ def main():
'from a target directory and produces an out file '
'with problems found that need to be addressed and '
'the json file which can be used to create the '
- 'blueprint.\n\nThis script only works with *-site.xml '
- 'files for now.'
- )
+ 'blueprint.\n\nThis script only works with *.xml *.yaml '
+ 'and *.properties extensions of files.')
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
default=False, help="output verbosity.")
@@ -187,6 +271,9 @@ def main():
metavar="FILE", help="Output directory. [default: /tmp]")
parser.add_option("-i", "--inputdir", dest="inputDir", help="Input directory.")
+ parser.add_option("-u", '--unknown-files-mapping-file',dest="unknown_files_mapping_file", default=None,
+ metavar="FILE", help=LOG4J_HELP_TEXT)
+
(options, args) = parser.parse_args()
# set verbose
@@ -198,6 +285,15 @@ def main():
ConfigMerge.INPUT_DIR = options.inputDir
ConfigMerge.OUTPUT_DIR = options.outputDir
+ #hack for logf4.properties files
+ if options.unknown_files_mapping_file:
+ ConfigMerge.UNKNOWN_FILES_MAPPING_FILE = options.unknown_files_mapping_file
+ with open(options.unknown_files_mapping_file) as f:
+ ConfigMerge.CONTENT_UNKNOWN_FILES_MAPPING_FILE = json.load(f)
+
+ if not os.path.exists(ConfigMerge.OUTPUT_DIR):
+ os.makedirs(ConfigMerge.OUTPUT_DIR)
+
logegr_file_name = os.path.join(ConfigMerge.OUTPUT_DIR, "takeover_config_merge.log")
file_handler = logging.FileHandler(logegr_file_name, mode="w")