You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2015/08/28 05:14:22 UTC
ambari git commit: AMBARI-12907: [PluggableStackDefinition] Add
support for overriding stack config properties (jluniya)
Repository: ambari
Updated Branches:
refs/heads/trunk 60d3a7613 -> b39288aab
AMBARI-12907: [PluggableStackDefinition] Add support for overriding stack config properties (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b39288aa
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b39288aa
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b39288aa
Branch: refs/heads/trunk
Commit: b39288aab9a8d92e35360c376f4b9da97d934c72
Parents: 60d3a76
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Thu Aug 27 20:14:16 2015 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Thu Aug 27 20:14:16 2015 -0700
----------------------------------------------------------------------
.../GenerateStackDefinition.py | 91 +++++
.../configs/SAPHD.json | 386 +++++++++++++++++++
2 files changed, 477 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b39288aa/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py b/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py
index c09c788..275501d 100644
--- a/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py
+++ b/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py
@@ -375,6 +375,91 @@ def process_py_files(file_path, config_data, stack_version_changes):
def process_xml_files(file_path, config_data, stack_version_changes):
return process_replacements(file_path, config_data, stack_version_changes)
+def process_config_xml(file_path, config_data):
+ tree = ET.parse(file_path)
+ root = tree.getroot()
+ #############################################################################################
+ # <resource_dir>/common-services/<service_name>/<service_version>/configuration/<config>.xml
+ #############################################################################################
+ COMMON_SERVICES_CONFIG_PATH_REGEX = r'common-services/([A-Za-z_-]+)/([0-9\.]+)/configuration/([A-Za-z0-9_-]+).xml'
+ #############################################################################################
+ # <resource_dir>/stacks/<stack_name>/<stack_version>/services/<service_name>/configuration/<config>.xml
+ #############################################################################################
+ STACK_SERVICE_CONFIG_PATH_REGEX = r'stacks/([A-Za-z_-]+)/([0-9\.]+)/services/([A-Za-z_-]+)/configuration/([A-Za-z0-9_-]+).xml'
+ #############################################################################################
+ # <resource_dir>/stacks/<stack_name>/<stack_version>/configuration/<config>.xml
+ #############################################################################################
+ STACK_CONFIG_PATH_REGEX = r'stacks/([A-Za-z_-]+)/([0-9\.]+)/configuration/([A-Za-z0-9_-]+).xml'
+
+ #########################################################################################
+ # Override stack config properties
+ #########################################################################################
+ match = re.search(COMMON_SERVICES_CONFIG_PATH_REGEX, file_path)
+ if match:
+ #############################################################################################
+ # Config file path in common services
+ #############################################################################################
+ path_service_name = match.group(1)
+ path_service_version = match.group(2)
+ path_config_name = match.group(3)
+ if 'common-services' in config_data:
+ for service in config_data['common-services']:
+ if service.name == path_service_name:
+ for serviceVersion in service.versions:
+ if serviceVersion.version == path_service_version:
+ if 'configurations' in serviceVersion:
+ for conf in serviceVersion['configurations']:
+ if conf.name == path_config_name:
+ for property_tag in root.findall('property'):
+ property_name = property_tag.find('name').text
+ if property_name in conf.properties:
+ value_tag = property_tag.find('value')
+ value_tag.text = conf.properties[property_name]
+ else:
+ match = re.search(STACK_SERVICE_CONFIG_PATH_REGEX, file_path)
+ if match:
+ #############################################################################################
+ # Config file path for a service in stack
+ #############################################################################################
+ path_stack_name = match.group(1)
+ path_stack_version = match.group(2)
+ path_service_name = match.group(3)
+ path_config_name = match.group(4)
+ for stack in config_data.versions:
+ if stack.version == path_stack_version:
+ for service in stack.services:
+ if service.name == path_service_name:
+ if 'configurations' in service:
+ for conf in service['configurations']:
+ if conf.name == path_config_name:
+ for property_tag in root.findall('property'):
+ property_name = property_tag.find('name').text
+ if property_name in conf.properties:
+ value_tag = property_tag.find('value')
+ value_tag.text = conf.properties[property_name]
+ else:
+ match = re.search(STACK_CONFIG_PATH_REGEX, file_path)
+ if match:
+ #############################################################################################
+ # Config file path for global stack configs
+ #############################################################################################
+ path_stack_name = match.group(1)
+ path_stack_version = match.group(2)
+ path_config_name = match.group(3)
+ for stack in config_data.versions:
+ if stack.version == path_stack_version:
+ if 'configurations' in stack:
+ for conf in stack['configurations']:
+ if conf.name == path_config_name:
+ for property_tag in root.findall('property'):
+ property_name = property_tag.find('name').text
+ if property_name in conf.properties:
+ value_tag = property_tag.find('value')
+ value_tag.text = conf.properties[property_name]
+
+ tree.write(file_path)
+ return file_path
+
class GeneratorHelper(object):
def __init__(self, config_data, resources_folder, output_folder):
self.config_data = config_data
@@ -415,6 +500,9 @@ class GeneratorHelper(object):
# process repoinfo.xml
if target.endswith('repoinfo.xml'):
target = process_repoinfo_xml(target, self.config_data, self.stack_version_changes, stack)
+ if os.path.basename(os.path.dirname(target)) == 'configuration':
+ # process configuration xml
+ target = process_config_xml(target, self.config_data)
# process upgrade-x.x.xml
_upgrade_re = re.compile('upgrade-(.*)\.xml')
result = re.search(_upgrade_re, target)
@@ -456,6 +544,9 @@ class GeneratorHelper(object):
# process metainfo.xml
if target.endswith('metainfo.xml'):
process_metainfo(target, self.config_data, self.stack_version_changes, parent_services)
+ if os.path.basename(os.path.dirname(target)) == 'configuration':
+ # process configuration xml
+ target = process_config_xml(target, self.config_data)
# process generic xml
if target.endswith('.xml'):
process_xml_files(target, self.config_data, self.stack_version_changes)
http://git-wip-us.apache.org/repos/asf/ambari/blob/b39288aa/ambari-common/src/main/python/pluggable_stack_definition/configs/SAPHD.json
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/pluggable_stack_definition/configs/SAPHD.json b/ambari-common/src/main/python/pluggable_stack_definition/configs/SAPHD.json
new file mode 100644
index 0000000..2ffa54d
--- /dev/null
+++ b/ambari-common/src/main/python/pluggable_stack_definition/configs/SAPHD.json
@@ -0,0 +1,386 @@
+{
+ "stackName": "SAPHD",
+ "baseStackName": "HDP",
+ "performCommonReplacements": true,
+ "textReplacements": [
+ ["hdp-select", "distro-select"]
+ ],
+ "preservedText": ["ext-2.2", "HDP-oozie", "hdp.version", "HDP_VERSION"],
+ "uiMapping": [
+ {
+ "stackVersionNumber": "2.3",
+ "sign": "<",
+ "baseStackFolder": "HDP2"
+ }
+ ],
+ "ambariProperties": {
+ "jdk.download.supported" : "false",
+ "jce.download.supported" : "false",
+ "jdk1.7.url" : "http://DOWNLOAD_NOT_SUPPORTED",
+ "jdk1.8.url" : "http://DOWNLOAD_NOT_SUPPORTED",
+ "jdk1.7.jcpol-url" : "http://DOWNLOAD_NOT_SUPPORTED",
+ "jdk1.8.jcpol-url" : "http://DOWNLOAD_NOT_SUPPORTED",
+ "rolling.upgrade.min.stack" : "SAPHD-2.2"
+ },
+ "common-services": [
+ {
+ "name": "HIVE",
+ "versions": [
+ {
+ "version": "0.12.0.2.0",
+ "configurations": [
+ {
+ "name": "hive-env",
+ "properties": {
+ "hive_database_type": "sqlanywhere",
+ "hive_database": "Existing SQLA Database"
+ }
+ },
+ {
+ "name": "hive-site",
+ "properties": {
+ "javax.jdo.option.ConnectionURL": "jdbc:sqlanywhere:host=localhost;database=hive",
+ "javax.jdo.option.ConnectionDriverName": "sap.jdbc4.sqlanywhere.IDriver"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "OOZIE",
+ "versions": [
+ {
+ "version": "4.0.0.2.0",
+ "configurations": [
+ {
+ "name": "oozie-env",
+ "properties": {
+ "oozie_database": "Existing SQLA Database"
+ }
+ },
+ {
+ "name": "oozie-site",
+ "properties": {
+ "oozie.service.JPAService.jdbc.url": "jdbc:sqlanywhere:host=localhost;database=oozie",
+ "oozie.service.JPAService.jdbc.driver": "sap.jdbc4.sqlanywhere.IDriver"
+ }
+ }
+ ]
+ },
+ {
+ "version": "4.2.0.2.3",
+ "configurations": [
+ {
+ "name": "oozie-site",
+ "properties": {
+ "oozie.service.JPAService.jdbc.url": "jdbc:sqlanywhere:host=localhost;database=oozie",
+ "oozie.service.JPAService.jdbc.driver": "sap.jdbc4.sqlanywhere.IDriver"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "RANGER",
+ "versions": [
+ {
+ "version": "0.4.0",
+ "configurations": [
+ {
+ "name": "admin-properties",
+ "properties": {
+ "DB_FLAVOR": "SQLA"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ],
+ "versions": [
+ {
+ "version": "2.0.6",
+ "baseVersion": "2.0.6",
+ "family": "redhat6,suse11",
+ "active": "false",
+ "services": [
+ {
+ "name": "KERBEROS"
+ },
+ {
+ "name": "AMBARI_METRICS"
+ },
+ {
+ "name": "HDFS"
+
+ },
+ {
+ "name": "ZOOKEEPER"
+ },
+ {
+ "name": "HBASE"
+ },
+ {
+ "name": "YARN"
+ },
+ {
+ "name": "MAPREDUCE2"
+ },
+ {
+ "name": "HIVE"
+ },
+ {
+ "name": "TEZ"
+ },
+ {
+ "name": "OOZIE"
+ },
+ {
+ "name": "KNOX"
+ },
+ {
+ "name": "PIG"
+ },
+ {
+ "name": "FLUME"
+ },
+ {
+ "name": "SQOOP"
+ }
+ ]
+ },
+ {
+ "version": "2.1",
+ "baseVersion": "2.1",
+ "active": "false",
+ "family": "redhat6,suse11",
+ "services": [
+ {
+ "name": "KERBEROS"
+ },
+ {
+ "name": "AMBARI_METRICS"
+ },
+ {
+ "name": "HDFS"
+ },
+ {
+ "name": "ZOOKEEPER"
+ },
+ {
+ "name": "HBASE"
+ },
+ {
+ "name": "YARN"
+ },
+ {
+ "name": "MAPREDUCE2"
+ },
+ {
+ "name": "HIVE"
+ },
+ {
+ "name": "TEZ"
+ },
+ {
+ "name": "OOZIE",
+ "configurations": [
+ {
+ "name": "oozie-site",
+ "properties": {
+ "oozie.service.JPAService.jdbc.url": "jdbc:sqlanywhere:host=localhost;database=oozie",
+ "oozie.service.JPAService.jdbc.driver": "sap.jdbc4.sqlanywhere.IDriver"
+ }
+ }
+ ]
+ },
+ {
+ "name": "KNOX"
+ },
+ {
+ "name": "PIG"
+ },
+ {
+ "name": "FLUME"
+ },
+ {
+ "name": "SQOOP"
+ },
+ {
+ "name": "FALCON"
+ },
+ {
+ "name": "STORM"
+ }
+ ]
+ },
+ {
+ "version": "2.2",
+ "baseVersion": "2.2",
+ "active": "false",
+ "family": "redhat6,suse11",
+ "services": [
+ {
+ "name": "KERBEROS"
+ },
+ {
+ "name": "AMBARI_METRICS"
+ },
+ {
+ "name": "HDFS"
+ },
+ {
+ "name": "ZOOKEEPER"
+ },
+ {
+ "name": "HBASE"
+ },
+ {
+ "name": "YARN"
+ },
+ {
+ "name": "MAPREDUCE2"
+ },
+ {
+ "name": "HIVE",
+ "configurations": [
+ {
+ "name": "hive-site",
+ "properties": {
+ "javax.jdo.option.ConnectionURL": "jdbc:sqlanywhere:host=localhost;database=hive",
+ "javax.jdo.option.ConnectionDriverName": "sap.jdbc4.sqlanywhere.IDriver"
+ }
+ }
+ ]
+ },
+ {
+ "name": "TEZ"
+ },
+ {
+ "name": "OOZIE"
+ },
+ {
+ "name": "KNOX"
+ },
+ {
+ "name": "PIG"
+ },
+ {
+ "name": "FLUME"
+ },
+ {
+ "name": "SQOOP"
+ },
+ {
+ "name": "FALCON"
+ },
+ {
+ "name": "STORM"
+ },
+ {
+ "name": "KAFKA"
+ },
+ {
+ "name": "RANGER"
+ },
+ {
+ "name": "SLIDER"
+ },
+ {
+ "name": "SPARK"
+ }
+ ]
+ },
+ {
+ "version": "2.3",
+ "baseVersion": "2.3",
+ "active": "true",
+ "family": "redhat6,redhat7,suse11",
+ "services": [
+ {
+ "name": "KERBEROS"
+ },
+ {
+ "name": "AMBARI_METRICS"
+ },
+ {
+ "name": "HDFS"
+ },
+ {
+ "name": "ZOOKEEPER"
+ },
+ {
+ "name": "HBASE"
+ },
+ {
+ "name": "YARN"
+ },
+ {
+ "name": "MAPREDUCE2"
+ },
+ {
+ "name": "HIVE"
+ },
+ {
+ "name": "TEZ"
+ },
+ {
+ "name": "OOZIE"
+ },
+ {
+ "name": "KNOX"
+ },
+ {
+ "name": "PIG"
+ },
+ {
+ "name": "FLUME"
+ },
+ {
+ "name": "SQOOP"
+ },
+ {
+ "name": "FALCON"
+ },
+ {
+ "name": "STORM"
+ },
+ {
+ "name": "KAFKA"
+ },
+ {
+ "name": "RANGER",
+ "configurations":[
+ {
+ "name": "ranger-admin-site",
+ "properties": {
+ "ranger.jpa.jdbc.driver": "sap.jdbc4.sqlanywhere.IDriver",
+ "ranger.jpa.jdbc.url": "jdbc:sqlanywhere:host=localhost;database=ranger"
+ }
+ }
+ ]
+ },
+ {
+ "name": "SLIDER"
+ },
+ {
+ "name": "SPARK"
+ },
+ {
+ "name": "RANGER_KMS"
+ },
+ {
+ "name": "ACCUMULO"
+ },
+ {
+ "name": "ATLAS"
+ },
+ {
+ "name": "MAHOUT"
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file