You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ma...@apache.org on 2016/08/17 05:38:43 UTC
[3/7] ambari git commit: Revert "[RTC-136620]: Redefine stack_advisor
logic for BigInsights stacks"
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b3f6d84/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_23.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_23.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_23.py
index cef0796..1eb2514 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_23.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_23.py
@@ -1,953 +1 @@
-#!/usr/bin/env ambari-python-wrap
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-import os
-import re
-import fnmatch
-import math
-import socket
-
-from stack_advisor_22 import *
-
-class HDP23StackAdvisor(HDP22StackAdvisor):
-
- def createComponentLayoutRecommendations(self, services, hosts):
- parentComponentLayoutRecommendations = super(HDP23StackAdvisor, self).createComponentLayoutRecommendations(services, hosts)
-
- # remove HAWQSTANDBY on a single node
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- if len(hostsList) == 1:
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- if "HAWQ" in servicesList:
- components = parentComponentLayoutRecommendations["blueprint"]["host_groups"][0]["components"]
- components = [ component for component in components if component["name"] != 'HAWQSTANDBY' ]
- parentComponentLayoutRecommendations["blueprint"]["host_groups"][0]["components"] = components
-
- return parentComponentLayoutRecommendations
-
- def getComponentLayoutValidations(self, services, hosts):
- parentItems = super(HDP23StackAdvisor, self).getComponentLayoutValidations(services, hosts)
-
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- componentsListList = [service["components"] for service in services["services"]]
- componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
- childItems = []
-
- if "SPARK" in servicesList:
- if "SPARK_THRIFTSERVER" in servicesList:
- if not "HIVE_SERVER" in servicesList:
- message = "SPARK_THRIFTSERVER requires HIVE services to be selected."
- childItems.append( {"type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER'} )
-
- hmsHosts = self.__getHosts(componentsList, "HIVE_METASTORE") if "HIVE" in servicesList else []
- sparkTsHosts = self.__getHosts(componentsList, "SPARK_THRIFTSERVER") if "SPARK" in servicesList else []
-
- # if Spark Thrift Server is deployed but no Hive Server is deployed
- if len(sparkTsHosts) > 0 and len(hmsHosts) == 0:
- message = "SPARK_THRIFTSERVER requires HIVE_METASTORE to be selected/deployed."
- childItems.append( { "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'SPARK_THRIFTSERVER' } )
-
- parentItems.extend(childItems)
- return parentItems
-
- def __getHosts(self, componentsList, componentName):
- host_lists = [component["hostnames"] for component in componentsList if
- component["component_name"] == componentName]
- if host_lists and len(host_lists) > 0:
- return host_lists[0]
- else:
- return []
-
- def getServiceConfigurationRecommenderDict(self):
- parentRecommendConfDict = super(HDP23StackAdvisor, self).getServiceConfigurationRecommenderDict()
- childRecommendConfDict = {
- "TEZ": self.recommendTezConfigurations,
- "HDFS": self.recommendHDFSConfigurations,
- "YARN": self.recommendYARNConfigurations,
- "HIVE": self.recommendHIVEConfigurations,
- "HBASE": self.recommendHBASEConfigurations,
- "KAFKA": self.recommendKAFKAConfigurations,
- "RANGER": self.recommendRangerConfigurations,
- "RANGER_KMS": self.recommendRangerKMSConfigurations,
- "FALCON": self.recommendFalconConfigurations,
- "STORM": self.recommendStormConfigurations,
- "SQOOP": self.recommendSqoopConfigurations
- }
- parentRecommendConfDict.update(childRecommendConfDict)
- return parentRecommendConfDict
-
- def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
- super(HDP23StackAdvisor, self).recommendTezConfigurations(configurations, clusterData, services, hosts)
-
- putTezProperty = self.putProperty(configurations, "tez-site")
- if "hive-site" in services["configurations"] and "hive.tez.container.size" in services["configurations"]["hive-site"]["properties"]:
- putTezProperty("tez.task.resource.memory.mb", services["configurations"]["hive-site"]["properties"]["hive.tez.container.size"])
- # remove 2gb limit for tez.runtime.io.sort.mb
- # in HDP 2.3 "tez.runtime.sorter.class" is set by default to PIPELINED, in other case comment calculation code below
- taskResourceMemory = int(configurations["tez-site"]["properties"]["tez.task.resource.memory.mb"])
- putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.4))
-
- if "tez-site" in services["configurations"] and "tez.runtime.sorter.class" in services["configurations"]["tez-site"]["properties"]:
- if services["configurations"]["tez-site"]["properties"]["tez.runtime.sorter.class"] == "LEGACY":
- putTezAttribute = self.putPropertyAttribute(configurations, "tez-site")
- putTezAttribute("tez.runtime.io.sort.mb", "maximum", 1800)
- pass
-
- serverProperties = services["ambari-server-properties"]
- latest_tez_jar_version = None
-
- server_host = socket.getfqdn()
- for host in hosts["items"]:
- if server_host == host["Hosts"]["host_name"]:
- server_host = host["Hosts"]["public_host_name"]
- server_port = '8080'
- server_protocol = 'http'
- views_dir = '/var/lib/ambari-server/resources/views/'
-
- if serverProperties:
- if 'client.api.port' in serverProperties:
- server_port = serverProperties['client.api.port']
- if 'views.dir' in serverProperties:
- views_dir = serverProperties['views.dir']
- if 'api.ssl' in serverProperties:
- if serverProperties['api.ssl'].lower() == 'true':
- server_protocol = 'https'
-
- views_work_dir = os.path.join(views_dir, 'work')
-
- if os.path.exists(views_work_dir) and os.path.isdir(views_work_dir):
- last_version = '0.0.0'
- for file in os.listdir(views_work_dir):
- if fnmatch.fnmatch(file, 'TEZ{*}'):
- current_version = file.lstrip("TEZ{").rstrip("}") # E.g.: TEZ{0.7.0.2.3.0.0-2154}
- if self.versionCompare(current_version.replace("-", "."), last_version.replace("-", ".")) >= 0:
- latest_tez_jar_version = current_version
- last_version = current_version
- pass
- pass
- pass
- pass
-
- if latest_tez_jar_version:
- tez_url = '{0}://{1}:{2}/#/main/views/TEZ/{3}/TEZ_CLUSTER_INSTANCE'.format(server_protocol, server_host, server_port, latest_tez_jar_version)
- putTezProperty("tez.tez-ui.history-url.base", tez_url)
- pass
-
- # TEZ JVM options
- jvmGCParams = "-XX:+UseParallelGC"
- if "ambari-server-properties" in services and "java.home" in services["ambari-server-properties"]:
- # JDK8 needs different parameters
- match = re.match(".*\/jdk(1\.\d+)[\-\_\.][^/]*$", services["ambari-server-properties"]["java.home"])
- if match and len(match.groups()) > 0:
- # Is version >= 1.8
- versionSplits = re.split("\.", match.group(1))
- if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7:
- jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB"
- putTezProperty('tez.am.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams)
- putTezProperty('tez.task.launch.cmd-opts', "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA " + jvmGCParams)
-
-
- def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
- super(HDP23StackAdvisor, self).recommendHBASEConfigurations(configurations, clusterData, services, hosts)
- putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
- putHbaseSitePropertyAttributes = self.putPropertyAttribute(configurations, "hbase-site")
- putHbaseEnvProperty = self.putProperty(configurations, "hbase-env", services)
- putHbaseEnvPropertyAttributes = self.putPropertyAttribute(configurations, "hbase-env")
-
- # bucket cache for 1.x is configured slightly differently, HBASE-11520
- threshold = 23 # 2 Gb is reserved for other offheap memory
- if (int(clusterData["hbaseRam"]) > threshold):
- # To enable cache - calculate values
- regionserver_total_ram = int(clusterData["hbaseRam"]) * 1024
- regionserver_heap_size = 20480
- regionserver_max_direct_memory_size = regionserver_total_ram - regionserver_heap_size
- hfile_block_cache_size = '0.4'
- block_cache_heap = 8192 # int(regionserver_heap_size * hfile_block_cache_size)
- hbase_regionserver_global_memstore_size = '0.4'
- reserved_offheap_memory = 2048
- bucketcache_offheap_memory = regionserver_max_direct_memory_size - reserved_offheap_memory
- hbase_bucketcache_size = bucketcache_offheap_memory
-
- # Set values in hbase-site
- putHbaseSiteProperty('hfile.block.cache.size', hfile_block_cache_size)
- putHbaseSiteProperty('hbase.regionserver.global.memstore.size', hbase_regionserver_global_memstore_size)
- putHbaseSiteProperty('hbase.bucketcache.ioengine', 'offheap')
- putHbaseSiteProperty('hbase.bucketcache.size', hbase_bucketcache_size)
- # 2.2 stack method was called earlier, unset
- putHbaseSitePropertyAttributes('hbase.bucketcache.percentage.in.combinedcache', 'delete', 'true')
-
- # Enable in hbase-env
- putHbaseEnvProperty('hbase_max_direct_memory_size', regionserver_max_direct_memory_size)
- putHbaseEnvProperty('hbase_regionserver_heapsize', regionserver_heap_size)
- else:
- # Disable
- putHbaseSitePropertyAttributes('hbase.bucketcache.ioengine', 'delete', 'true')
- putHbaseSitePropertyAttributes('hbase.bucketcache.size', 'delete', 'true')
- putHbaseSitePropertyAttributes('hbase.bucketcache.percentage.in.combinedcache', 'delete', 'true')
-
- putHbaseEnvPropertyAttributes('hbase_max_direct_memory_size', 'delete', 'true')
-
- if 'hbase-env' in services['configurations'] and 'phoenix_sql_enabled' in services['configurations']['hbase-env']['properties'] and \
- 'true' == services['configurations']['hbase-env']['properties']['phoenix_sql_enabled'].lower():
- putHbaseSiteProperty("hbase.rpc.controllerfactory.class", "org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory")
- putHbaseSiteProperty("hbase.region.server.rpc.scheduler.factory.class", "org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory")
- else:
- putHbaseSitePropertyAttributes('hbase.region.server.rpc.scheduler.factory.class', 'delete', 'true')
-
-
- def recommendHIVEConfigurations(self, configurations, clusterData, services, hosts):
- super(HDP23StackAdvisor, self).recommendHIVEConfigurations(configurations, clusterData, services, hosts)
- putHiveSiteProperty = self.putProperty(configurations, "hive-site", services)
- putHiveServerProperty = self.putProperty(configurations, "hiveserver2-site", services)
- putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site")
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- # hive_security_authorization == 'ranger'
- if str(configurations["hive-env"]["properties"]["hive_security_authorization"]).lower() == "ranger":
- putHiveServerProperty("hive.security.authorization.manager", "org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory")
-
- # TEZ JVM options
- jvmGCParams = "-XX:+UseParallelGC"
- if "ambari-server-properties" in services and "java.home" in services["ambari-server-properties"]:
- # JDK8 needs different parameters
- match = re.match(".*\/jdk(1\.\d+)[\-\_\.][^/]*$", services["ambari-server-properties"]["java.home"])
- if match and len(match.groups()) > 0:
- # Is version >= 1.8
- versionSplits = re.split("\.", match.group(1))
- if versionSplits and len(versionSplits) > 1 and int(versionSplits[0]) > 0 and int(versionSplits[1]) > 7:
- jvmGCParams = "-XX:+UseG1GC -XX:+ResizeTLAB"
- putHiveSiteProperty('hive.tez.java.opts', "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA " + jvmGCParams + " -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps")
-
- # if hive using sqla db, then we should add DataNucleus property
- sqla_db_used = 'hive-env' in services['configurations'] and 'hive_database' in services['configurations']['hive-env']['properties'] and \
- services['configurations']['hive-env']['properties']['hive_database'] == 'Existing SQL Anywhere Database'
- if sqla_db_used:
- putHiveSiteProperty('datanucleus.rdbms.datastoreAdapterClassName','org.datanucleus.store.rdbms.adapter.SQLAnywhereAdapter')
- else:
- putHiveSitePropertyAttribute('datanucleus.rdbms.datastoreAdapterClassName', 'delete', 'true')
-
- # atlas
- hooks_property = "hive.exec.post.hooks"
- if hooks_property in configurations["hive-site"]["properties"]:
- hooks_value = configurations["hive-site"]["properties"][hooks_property]
- else:
- hooks_value = " "
-
- include_atlas = "ATLAS" in servicesList
- atlas_hook_class = "org.apache.atlas.hive.hook.HiveHook"
- if include_atlas and atlas_hook_class not in hooks_value:
- if hooks_value == " ":
- hooks_value = atlas_hook_class
- else:
- hooks_value = hooks_value + "," + atlas_hook_class
- if not include_atlas and atlas_hook_class in hooks_value:
- hooks_classes = []
- for hook_class in hooks_value.split(","):
- if hook_class != atlas_hook_class and hook_class != " ":
- hooks_classes.append(hook_class)
- if hooks_classes:
- hooks_value = ",".join(hooks_classes)
- else:
- hooks_value = " "
- putHiveSiteProperty(hooks_property, hooks_value)
-
- atlas_server_host_info = self.getHostWithComponent("ATLAS", "ATLAS_SERVER", services, hosts)
- if include_atlas and atlas_server_host_info:
- atlas_rest_host = atlas_server_host_info['Hosts']['host_name']
- scheme = "http"
- metadata_port = "21000"
- if 'application-properties' in services['configurations']:
- tls_enabled = services['configurations']['application-properties']['properties']['atlas.enableTLS']
- metadata_port = services['configurations']['application-properties']['properties']['atlas.server.http.port']
- if tls_enabled.lower() == "true":
- scheme = "https"
- metadata_port = services['configurations']['application-properties']['properties']['atlas.server.https.port']
- putHiveSiteProperty('atlas.rest.address', '{0}://{1}:{2}'.format(scheme, atlas_rest_host, metadata_port))
- else:
- putHiveSitePropertyAttribute('atlas.cluster.name', 'delete', 'true')
- putHiveSitePropertyAttribute('atlas.rest.address', 'delete', 'true')
-
- def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
- super(HDP23StackAdvisor, self).recommendHDFSConfigurations(configurations, clusterData, services, hosts)
-
- putHdfsSiteProperty = self.putProperty(configurations, "hdfs-site", services)
- putHdfsSitePropertyAttribute = self.putPropertyAttribute(configurations, "hdfs-site")
-
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- if "HAWQ" in servicesList:
- # Set dfs.allow.truncate to true
- putHdfsSiteProperty('dfs.allow.truncate', 'true')
-
- if ('ranger-hdfs-plugin-properties' in services['configurations']) and ('ranger-hdfs-plugin-enabled' in services['configurations']['ranger-hdfs-plugin-properties']['properties']):
- rangerPluginEnabled = ''
- if 'ranger-hdfs-plugin-properties' in configurations and 'ranger-hdfs-plugin-enabled' in configurations['ranger-hdfs-plugin-properties']['properties']:
- rangerPluginEnabled = configurations['ranger-hdfs-plugin-properties']['properties']['ranger-hdfs-plugin-enabled']
- elif 'ranger-hdfs-plugin-properties' in services['configurations'] and 'ranger-hdfs-plugin-enabled' in services['configurations']['ranger-hdfs-plugin-properties']['properties']:
- rangerPluginEnabled = services['configurations']['ranger-hdfs-plugin-properties']['properties']['ranger-hdfs-plugin-enabled']
-
- if rangerPluginEnabled and (rangerPluginEnabled.lower() == 'Yes'.lower()):
- putHdfsSiteProperty("dfs.namenode.inode.attributes.provider.class",'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer')
- else:
- putHdfsSitePropertyAttribute('dfs.namenode.inode.attributes.provider.class', 'delete', 'true')
- else:
- putHdfsSitePropertyAttribute('dfs.namenode.inode.attributes.provider.class', 'delete', 'true')
-
- def recommendKAFKAConfigurations(self, configurations, clusterData, services, hosts):
- kafka_broker = getServicesSiteProperties(services, "kafka-broker")
-
- # kerberos security for kafka is decided from `security.inter.broker.protocol` property value
- security_enabled = (kafka_broker is not None and 'security.inter.broker.protocol' in kafka_broker
- and 'SASL' in kafka_broker['security.inter.broker.protocol'])
- putKafkaBrokerProperty = self.putProperty(configurations, "kafka-broker", services)
- putKafkaLog4jProperty = self.putProperty(configurations, "kafka-log4j", services)
- putKafkaBrokerAttributes = self.putPropertyAttribute(configurations, "kafka-broker")
-
- #If AMS is part of Services, use the KafkaTimelineMetricsReporter for metric reporting. Default is ''.
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- if "AMBARI_METRICS" in servicesList:
- putKafkaBrokerProperty('kafka.metrics.reporters', 'org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter')
-
- if "ranger-env" in services["configurations"] and "ranger-kafka-plugin-properties" in services["configurations"] and \
- "ranger-kafka-plugin-enabled" in services["configurations"]["ranger-env"]["properties"]:
- putKafkaRangerPluginProperty = self.putProperty(configurations, "ranger-kafka-plugin-properties", services)
- rangerEnvKafkaPluginProperty = services["configurations"]["ranger-env"]["properties"]["ranger-kafka-plugin-enabled"]
- putKafkaRangerPluginProperty("ranger-kafka-plugin-enabled", rangerEnvKafkaPluginProperty)
-
- if 'ranger-kafka-plugin-properties' in services['configurations'] and ('ranger-kafka-plugin-enabled' in services['configurations']['ranger-kafka-plugin-properties']['properties']):
- kafkaLog4jRangerLines = [{
- "name": "log4j.appender.rangerAppender",
- "value": "org.apache.log4j.DailyRollingFileAppender"
- },
- {
- "name": "log4j.appender.rangerAppender.DatePattern",
- "value": "'.'yyyy-MM-dd-HH"
- },
- {
- "name": "log4j.appender.rangerAppender.File",
- "value": "${kafka.logs.dir}/ranger_kafka.log"
- },
- {
- "name": "log4j.appender.rangerAppender.layout",
- "value": "org.apache.log4j.PatternLayout"
- },
- {
- "name": "log4j.appender.rangerAppender.layout.ConversionPattern",
- "value": "%d{ISO8601} %p [%t] %C{6} (%F:%L) - %m%n"
- },
- {
- "name": "log4j.logger.org.apache.ranger",
- "value": "INFO, rangerAppender"
- }]
-
- rangerPluginEnabled=''
- if 'ranger-kafka-plugin-properties' in configurations and 'ranger-kafka-plugin-enabled' in configurations['ranger-kafka-plugin-properties']['properties']:
- rangerPluginEnabled = configurations['ranger-kafka-plugin-properties']['properties']['ranger-kafka-plugin-enabled']
- elif 'ranger-kafka-plugin-properties' in services['configurations'] and 'ranger-kafka-plugin-enabled' in services['configurations']['ranger-kafka-plugin-properties']['properties']:
- rangerPluginEnabled = services['configurations']['ranger-kafka-plugin-properties']['properties']['ranger-kafka-plugin-enabled']
-
- if rangerPluginEnabled and rangerPluginEnabled.lower() == "Yes".lower():
- # recommend authorizer.class.name
- putKafkaBrokerProperty("authorizer.class.name", 'org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer')
- # change kafka-log4j when ranger plugin is installed
-
- if 'kafka-log4j' in services['configurations'] and 'content' in services['configurations']['kafka-log4j']['properties']:
- kafkaLog4jContent = services['configurations']['kafka-log4j']['properties']['content']
- for item in range(len(kafkaLog4jRangerLines)):
- if kafkaLog4jRangerLines[item]["name"] not in kafkaLog4jContent:
- kafkaLog4jContent+= '\n' + kafkaLog4jRangerLines[item]["name"] + '=' + kafkaLog4jRangerLines[item]["value"]
- putKafkaLog4jProperty("content",kafkaLog4jContent)
-
-
- else:
- # Kerberized Cluster with Ranger plugin disabled
- if security_enabled and 'kafka-broker' in services['configurations'] and 'authorizer.class.name' in services['configurations']['kafka-broker']['properties'] and \
- services['configurations']['kafka-broker']['properties']['authorizer.class.name'] == 'org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer':
- putKafkaBrokerProperty("authorizer.class.name", 'kafka.security.auth.SimpleAclAuthorizer')
- # Non-kerberos Cluster with Ranger plugin disabled
- else:
- putKafkaBrokerAttributes('authorizer.class.name', 'delete', 'true')
-
- # Non-Kerberos Cluster without Ranger
- elif not security_enabled:
- putKafkaBrokerAttributes('authorizer.class.name', 'delete', 'true')
-
- def recommendRangerKMSConfigurations(self, configurations, clusterData, services, hosts):
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- putRangerKmsDbksProperty = self.putProperty(configurations, "dbks-site", services)
- putRangerKmsProperty = self.putProperty(configurations, "kms-properties", services)
- kmsEnvProperties = getSiteProperties(services['configurations'], 'kms-env')
- putCoreSiteProperty = self.putProperty(configurations, "core-site", services)
- putCoreSitePropertyAttribute = self.putPropertyAttribute(configurations, "core-site")
- putRangerKmsAuditProperty = self.putProperty(configurations, "ranger-kms-audit", services)
-
- if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']):
-
- rangerKmsDbFlavor = services['configurations']["kms-properties"]["properties"]["DB_FLAVOR"]
-
- if ('db_host' in services['configurations']['kms-properties']['properties']) and ('db_name' in services['configurations']['kms-properties']['properties']):
-
- rangerKmsDbHost = services['configurations']["kms-properties"]["properties"]["db_host"]
- rangerKmsDbName = services['configurations']["kms-properties"]["properties"]["db_name"]
-
- ranger_kms_db_url_dict = {
- 'MYSQL': {'ranger.ks.jpa.jdbc.driver': 'com.mysql.jdbc.Driver',
- 'ranger.ks.jpa.jdbc.url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/' + rangerKmsDbName},
- 'ORACLE': {'ranger.ks.jpa.jdbc.driver': 'oracle.jdbc.driver.OracleDriver',
- 'ranger.ks.jpa.jdbc.url': 'jdbc:oracle:thin:@' + self.getOracleDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost, rangerKmsDbName)},
- 'POSTGRES': {'ranger.ks.jpa.jdbc.driver': 'org.postgresql.Driver',
- 'ranger.ks.jpa.jdbc.url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/' + rangerKmsDbName},
- 'MSSQL': {'ranger.ks.jpa.jdbc.driver': 'com.microsoft.sqlserver.jdbc.SQLServerDriver',
- 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';databaseName=' + rangerKmsDbName},
- 'SQLA': {'ranger.ks.jpa.jdbc.driver': 'sap.jdbc4.sqlanywhere.IDriver',
- 'ranger.ks.jpa.jdbc.url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';database=' + rangerKmsDbName}
- }
-
- rangerKmsDbProperties = ranger_kms_db_url_dict.get(rangerKmsDbFlavor, ranger_kms_db_url_dict['MYSQL'])
- for key in rangerKmsDbProperties:
- putRangerKmsDbksProperty(key, rangerKmsDbProperties.get(key))
-
- if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList:
- kmsUser = kmsEnvProperties['kms_user']
- kmsUserOld = getOldValue(self, services, 'kms-env', 'kms_user')
- putCoreSiteProperty('hadoop.proxyuser.{0}.groups'.format(kmsUser), '*')
- if kmsUserOld is not None and kmsUser != kmsUserOld:
- putCoreSitePropertyAttribute("hadoop.proxyuser.{0}.groups".format(kmsUserOld), 'delete', 'true')
- services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUserOld)})
- services["forced-configurations"].append({"type" : "core-site", "name" : "hadoop.proxyuser.{0}.groups".format(kmsUser)})
-
- if "HDFS" in servicesList:
- if 'core-site' in services['configurations'] and ('fs.defaultFS' in services['configurations']['core-site']['properties']):
- default_fs = services['configurations']['core-site']['properties']['fs.defaultFS']
- putRangerKmsAuditProperty('xasecure.audit.destination.hdfs.dir', '{0}/{1}/{2}'.format(default_fs,'ranger','audit'))
-
-
- def getOracleDBConnectionHostPort(self, db_type, db_host, rangerDbName):
- connection_string = self.getDBConnectionHostPort(db_type, db_host)
- colon_count = db_host.count(':')
- if colon_count == 1 and '/' in db_host:
- connection_string = "//" + connection_string
- elif colon_count == 0 or colon_count == 1:
- connection_string = "//" + connection_string + "/" + rangerDbName if rangerDbName else "//" + connection_string
-
- return connection_string
-
- def getDBConnectionHostPort(self, db_type, db_host):
- connection_string = ""
- if db_type is None or db_type == "":
- return connection_string
- else:
- colon_count = db_host.count(':')
- if colon_count == 0:
- if DB_TYPE_DEFAULT_PORT_MAP.has_key(db_type):
- connection_string = db_host + ":" + DB_TYPE_DEFAULT_PORT_MAP[db_type]
- else:
- connection_string = db_host
- elif colon_count == 1:
- connection_string = db_host
- elif colon_count == 2:
- connection_string = db_host
-
- return connection_string
-
-
- def recommendRangerConfigurations(self, configurations, clusterData, services, hosts):
- super(HDP23StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts)
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- putRangerAdminProperty = self.putProperty(configurations, "ranger-admin-site", services)
- putRangerEnvProperty = self.putProperty(configurations, "ranger-env", services)
- putRangerUgsyncSite = self.putProperty(configurations, "ranger-ugsync-site", services)
-
- if 'admin-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['admin-properties']['properties'])\
- and ('db_host' in services['configurations']['admin-properties']['properties']) and ('db_name' in services['configurations']['admin-properties']['properties']):
-
- rangerDbFlavor = services['configurations']["admin-properties"]["properties"]["DB_FLAVOR"]
- rangerDbHost = services['configurations']["admin-properties"]["properties"]["db_host"]
- rangerDbName = services['configurations']["admin-properties"]["properties"]["db_name"]
- ranger_db_url_dict = {
- 'MYSQL': {'ranger.jpa.jdbc.driver': 'com.mysql.jdbc.Driver',
- 'ranger.jpa.jdbc.url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/' + rangerDbName},
- 'ORACLE': {'ranger.jpa.jdbc.driver': 'oracle.jdbc.driver.OracleDriver',
- 'ranger.jpa.jdbc.url': 'jdbc:oracle:thin:@' + self.getOracleDBConnectionHostPort(rangerDbFlavor, rangerDbHost, rangerDbName)},
- 'POSTGRES': {'ranger.jpa.jdbc.driver': 'org.postgresql.Driver',
- 'ranger.jpa.jdbc.url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/' + rangerDbName},
- 'MSSQL': {'ranger.jpa.jdbc.driver': 'com.microsoft.sqlserver.jdbc.SQLServerDriver',
- 'ranger.jpa.jdbc.url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';databaseName=' + rangerDbName},
- 'SQLA': {'ranger.jpa.jdbc.driver': 'sap.jdbc4.sqlanywhere.IDriver',
- 'ranger.jpa.jdbc.url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';database=' + rangerDbName}
- }
- rangerDbProperties = ranger_db_url_dict.get(rangerDbFlavor, ranger_db_url_dict['MYSQL'])
- for key in rangerDbProperties:
- putRangerAdminProperty(key, rangerDbProperties.get(key))
-
- if 'admin-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['admin-properties']['properties']) \
- and ('db_host' in services['configurations']['admin-properties']['properties']):
-
- rangerDbFlavor = services['configurations']["admin-properties"]["properties"]["DB_FLAVOR"]
- rangerDbHost = services['configurations']["admin-properties"]["properties"]["db_host"]
- ranger_db_privelege_url_dict = {
- 'MYSQL': {'ranger_privelege_user_jdbc_url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost)},
- 'ORACLE': {'ranger_privelege_user_jdbc_url': 'jdbc:oracle:thin:@' + self.getOracleDBConnectionHostPort(rangerDbFlavor, rangerDbHost, None)},
- 'POSTGRES': {'ranger_privelege_user_jdbc_url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + '/postgres'},
- 'MSSQL': {'ranger_privelege_user_jdbc_url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';'},
- 'SQLA': {'ranger_privelege_user_jdbc_url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerDbFlavor, rangerDbHost) + ';'}
- }
- rangerPrivelegeDbProperties = ranger_db_privelege_url_dict.get(rangerDbFlavor, ranger_db_privelege_url_dict['MYSQL'])
- for key in rangerPrivelegeDbProperties:
- putRangerEnvProperty(key, rangerPrivelegeDbProperties.get(key))
-
- # Recommend ldap settings based on ambari.properties configuration
- if 'ambari-server-properties' in services and \
- 'ambari.ldap.isConfigured' in services['ambari-server-properties'] and \
- services['ambari-server-properties']['ambari.ldap.isConfigured'].lower() == "true":
- serverProperties = services['ambari-server-properties']
- if 'authentication.ldap.baseDn' in serverProperties:
- putRangerUgsyncSite('ranger.usersync.ldap.searchBase', serverProperties['authentication.ldap.baseDn'])
- if 'authentication.ldap.groupMembershipAttr' in serverProperties:
- putRangerUgsyncSite('ranger.usersync.group.memberattributename', serverProperties['authentication.ldap.groupMembershipAttr'])
- if 'authentication.ldap.groupNamingAttr' in serverProperties:
- putRangerUgsyncSite('ranger.usersync.group.nameattribute', serverProperties['authentication.ldap.groupNamingAttr'])
- if 'authentication.ldap.groupObjectClass' in serverProperties:
- putRangerUgsyncSite('ranger.usersync.group.objectclass', serverProperties['authentication.ldap.groupObjectClass'])
- if 'authentication.ldap.managerDn' in serverProperties:
- putRangerUgsyncSite('ranger.usersync.ldap.binddn', serverProperties['authentication.ldap.managerDn'])
- if 'authentication.ldap.primaryUrl' in serverProperties:
- ldap_protocol = 'ldap://'
- if 'authentication.ldap.useSSL' in serverProperties and serverProperties['authentication.ldap.useSSL'] == 'true':
- ldap_protocol = 'ldaps://'
- ldapUrl = ldap_protocol + serverProperties['authentication.ldap.primaryUrl'] if serverProperties['authentication.ldap.primaryUrl'] else serverProperties['authentication.ldap.primaryUrl']
- putRangerUgsyncSite('ranger.usersync.ldap.url', ldapUrl)
- if 'authentication.ldap.userObjectClass' in serverProperties:
- putRangerUgsyncSite('ranger.usersync.ldap.user.objectclass', serverProperties['authentication.ldap.userObjectClass'])
- if 'authentication.ldap.usernameAttribute' in serverProperties:
- putRangerUgsyncSite('ranger.usersync.ldap.user.nameattribute', serverProperties['authentication.ldap.usernameAttribute'])
-
-
- # Recommend Ranger Authentication method
- authMap = {
- 'org.apache.ranger.unixusersync.process.UnixUserGroupBuilder': 'UNIX',
- 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': 'LDAP'
- }
-
- if 'ranger-ugsync-site' in services['configurations'] and 'ranger.usersync.source.impl.class' in services['configurations']["ranger-ugsync-site"]["properties"]:
- rangerUserSyncClass = services['configurations']["ranger-ugsync-site"]["properties"]["ranger.usersync.source.impl.class"]
- if rangerUserSyncClass in authMap:
- rangerSqlConnectorProperty = authMap.get(rangerUserSyncClass)
- putRangerAdminProperty('ranger.authentication.method', rangerSqlConnectorProperty)
-
-
- if 'ranger-env' in services['configurations'] and 'is_solrCloud_enabled' in services['configurations']["ranger-env"]["properties"]:
- isSolrCloudEnabled = services['configurations']["ranger-env"]["properties"]["is_solrCloud_enabled"] == "true"
- else:
- isSolrCloudEnabled = False
-
- if isSolrCloudEnabled:
- zookeeper_host_port = self.getZKHostPortString(services)
- ranger_audit_zk_port = ''
- if zookeeper_host_port:
- ranger_audit_zk_port = '{0}/{1}'.format(zookeeper_host_port, 'ranger_audits')
- putRangerAdminProperty('ranger.audit.solr.zookeepers', ranger_audit_zk_port)
- else:
- putRangerAdminProperty('ranger.audit.solr.zookeepers', 'NONE')
-
- # Recommend ranger.audit.solr.zookeepers and xasecure.audit.destination.hdfs.dir
- include_hdfs = "HDFS" in servicesList
- if include_hdfs:
- if 'core-site' in services['configurations'] and ('fs.defaultFS' in services['configurations']['core-site']['properties']):
- default_fs = services['configurations']['core-site']['properties']['fs.defaultFS']
- putRangerEnvProperty('xasecure.audit.destination.hdfs.dir', '{0}/{1}/{2}'.format(default_fs,'ranger','audit'))
-
- # Recommend Ranger supported service's audit properties
- ranger_services = [
- {'service_name': 'HDFS', 'audit_file': 'ranger-hdfs-audit'},
- {'service_name': 'YARN', 'audit_file': 'ranger-yarn-audit'},
- {'service_name': 'HBASE', 'audit_file': 'ranger-hbase-audit'},
- {'service_name': 'HIVE', 'audit_file': 'ranger-hive-audit'},
- {'service_name': 'KNOX', 'audit_file': 'ranger-knox-audit'},
- {'service_name': 'KAFKA', 'audit_file': 'ranger-kafka-audit'},
- {'service_name': 'STORM', 'audit_file': 'ranger-storm-audit'}
- ]
-
- for item in range(len(ranger_services)):
- if ranger_services[item]['service_name'] in servicesList:
- component_audit_file = ranger_services[item]['audit_file']
- if component_audit_file in services["configurations"]:
- ranger_audit_dict = [
- {'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.db', 'target_configname': 'xasecure.audit.destination.db'},
- {'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.hdfs', 'target_configname': 'xasecure.audit.destination.hdfs'},
- {'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.hdfs.dir', 'target_configname': 'xasecure.audit.destination.hdfs.dir'},
- {'filename': 'ranger-env', 'configname': 'xasecure.audit.destination.solr', 'target_configname': 'xasecure.audit.destination.solr'},
- {'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.urls', 'target_configname': 'xasecure.audit.destination.solr.urls'},
- {'filename': 'ranger-admin-site', 'configname': 'ranger.audit.solr.zookeepers', 'target_configname': 'xasecure.audit.destination.solr.zookeepers'}
- ]
- putRangerAuditProperty = self.putProperty(configurations, component_audit_file, services)
-
- for item in ranger_audit_dict:
- if item['filename'] in services["configurations"] and item['configname'] in services["configurations"][item['filename']]["properties"]:
- if item['filename'] in configurations and item['configname'] in configurations[item['filename']]["properties"]:
- rangerAuditProperty = configurations[item['filename']]["properties"][item['configname']]
- else:
- rangerAuditProperty = services["configurations"][item['filename']]["properties"][item['configname']]
- putRangerAuditProperty(item['target_configname'], rangerAuditProperty)
-
- audit_solr_flag = 'false'
- audit_db_flag = 'false'
- ranger_audit_source_type = 'solr'
- if 'ranger-env' in services['configurations'] and 'xasecure.audit.destination.solr' in services['configurations']["ranger-env"]["properties"]:
- audit_solr_flag = services['configurations']["ranger-env"]["properties"]['xasecure.audit.destination.solr']
- if 'ranger-env' in services['configurations'] and 'xasecure.audit.destination.db' in services['configurations']["ranger-env"]["properties"]:
- audit_db_flag = services['configurations']["ranger-env"]["properties"]['xasecure.audit.destination.db']
-
- if audit_db_flag == 'true' and audit_solr_flag == 'false':
- ranger_audit_source_type = 'db'
- putRangerAdminProperty('ranger.audit.source.type',ranger_audit_source_type)
-
- knox_host = 'localhost'
- knox_port = '8443'
- if 'KNOX' in servicesList:
- knox_hosts = self.getComponentHostNames(services, "KNOX", "KNOX_GATEWAY")
- knox_host = knox_hosts[0]
- if 'gateway-site' in services['configurations'] and 'gateway.port' in services['configurations']["gateway-site"]["properties"]:
- knox_port = services['configurations']["gateway-site"]["properties"]['gateway.port']
- putRangerAdminProperty('ranger.sso.providerurl', 'https://{0}:{1}/gateway/knoxsso/api/v1/websso'.format(knox_host, knox_port))
-
-
- def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
- super(HDP23StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts)
- putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services)
- putYarnSitePropertyAttributes = self.putPropertyAttribute(configurations, "yarn-site")
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
-
- if "tez-site" not in services["configurations"]:
- putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', '')
- else:
- putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', 'org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl')
-
- if "ranger-env" in services["configurations"] and "ranger-yarn-plugin-properties" in services["configurations"] and \
- "ranger-yarn-plugin-enabled" in services["configurations"]["ranger-env"]["properties"]:
- putYarnRangerPluginProperty = self.putProperty(configurations, "ranger-yarn-plugin-properties", services)
- rangerEnvYarnPluginProperty = services["configurations"]["ranger-env"]["properties"]["ranger-yarn-plugin-enabled"]
- putYarnRangerPluginProperty("ranger-yarn-plugin-enabled", rangerEnvYarnPluginProperty)
- rangerPluginEnabled = ''
- if 'ranger-yarn-plugin-properties' in configurations and 'ranger-yarn-plugin-enabled' in configurations['ranger-yarn-plugin-properties']['properties']:
- rangerPluginEnabled = configurations['ranger-yarn-plugin-properties']['properties']['ranger-yarn-plugin-enabled']
- elif 'ranger-yarn-plugin-properties' in services['configurations'] and 'ranger-yarn-plugin-enabled' in services['configurations']['ranger-yarn-plugin-properties']['properties']:
- rangerPluginEnabled = services['configurations']['ranger-yarn-plugin-properties']['properties']['ranger-yarn-plugin-enabled']
-
- if rangerPluginEnabled and (rangerPluginEnabled.lower() == 'Yes'.lower()):
- putYarnSiteProperty('yarn.acl.enable','true')
- putYarnSiteProperty('yarn.authorization-provider','org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer')
- else:
- putYarnSitePropertyAttributes('yarn.authorization-provider', 'delete', 'true')
-
- if 'RANGER_KMS' in servicesList and 'KERBEROS' in servicesList:
- if 'yarn-site' in services["configurations"] and 'yarn.resourcemanager.proxy-user-privileges.enabled' in services["configurations"]["yarn-site"]["properties"]:
- putYarnSiteProperty('yarn.resourcemanager.proxy-user-privileges.enabled', 'false')
-
-
- def recommendSqoopConfigurations(self, configurations, clusterData, services, hosts):
- putSqoopSiteProperty = self.putProperty(configurations, "sqoop-site", services)
-
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- if "ATLAS" in servicesList:
- putSqoopSiteProperty('sqoop.job.data.publish.class', 'org.apache.atlas.sqoop.hook.SqoopHook')
-
- def recommendStormConfigurations(self, configurations, clusterData, services, hosts):
- super(HDP23StackAdvisor, self).recommendStormConfigurations(configurations, clusterData, services, hosts)
- putStormStartupProperty = self.putProperty(configurations, "storm-site", services)
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
-
- if "storm-site" in services["configurations"]:
- # atlas
- notifier_plugin_property = "storm.topology.submission.notifier.plugin.class"
- if notifier_plugin_property in services["configurations"]["storm-site"]["properties"]:
- notifier_plugin_value = services["configurations"]["storm-site"]["properties"][notifier_plugin_property]
- if notifier_plugin_value is None:
- notifier_plugin_value = " "
- else:
- notifier_plugin_value = " "
-
- include_atlas = "ATLAS" in servicesList
- atlas_hook_class = "org.apache.atlas.storm.hook.StormAtlasHook"
- if include_atlas and atlas_hook_class not in notifier_plugin_value:
- if notifier_plugin_value == " ":
- notifier_plugin_value = atlas_hook_class
- else:
- notifier_plugin_value = notifier_plugin_value + "," + atlas_hook_class
- if not include_atlas and atlas_hook_class in notifier_plugin_value:
- application_classes = []
- for application_class in notifier_plugin_value.split(","):
- if application_class != atlas_hook_class and application_class != " ":
- application_classes.append(application_class)
- if application_classes:
- notifier_plugin_value = ",".join(application_classes)
- else:
- notifier_plugin_value = " "
-
- if notifier_plugin_value != " ":
- putStormStartupProperty(notifier_plugin_property, notifier_plugin_value)
-
- def recommendFalconConfigurations(self, configurations, clusterData, services, hosts):
- putFalconStartupProperty = self.putProperty(configurations, "falcon-startup.properties", services)
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
-
- # atlas
- application_services_property = "*.application.services"
- if "falcon-startup.properties" in services["configurations"] and application_services_property in services["configurations"]["falcon-startup.properties"]["properties"]:
- application_services_value = services["configurations"]["falcon-startup.properties"]["properties"][application_services_property]
- else:
- application_services_value = " "
-
- include_atlas = "ATLAS" in servicesList
- atlas_application_class = "org.apache.falcon.atlas.service.AtlasService"
- if include_atlas and atlas_application_class not in application_services_value:
- if application_services_value == " ":
- application_services_value = atlas_application_class
- else:
- application_services_value = application_services_value + "," + atlas_application_class
- if not include_atlas and atlas_application_class in application_services_value:
- application_classes = []
- for application_class in application_services_value.split(","):
- if application_class != atlas_application_class and application_class != " ":
- application_classes.append(application_class)
- if application_classes:
- application_services_value = ",".join(application_classes)
- else:
- application_services_value = " "
- putFalconStartupProperty(application_services_property, application_services_value)
-
- def getServiceConfigurationValidators(self):
- parentValidators = super(HDP23StackAdvisor, self).getServiceConfigurationValidators()
- childValidators = {
- "HDFS": {"hdfs-site": self.validateHDFSConfigurations},
- "HIVE": {"hiveserver2-site": self.validateHiveServer2Configurations,
- "hive-site": self.validateHiveConfigurations},
- "HBASE": {"hbase-site": self.validateHBASEConfigurations},
- "KAKFA": {"kafka-broker": self.validateKAFKAConfigurations},
- "YARN": {"yarn-site": self.validateYARNConfigurations},
- "RANGER": {"admin-properties": self.validateRangerAdminConfigurations}
- }
- self.mergeValidators(parentValidators, childValidators)
- return parentValidators
-
- def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- parentValidationProblems = super(HDP23StackAdvisor, self).validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
-
- # We can not access property hadoop.security.authentication from the
- # other config (core-site). That's why we are using another heuristics here
- hdfs_site = properties
- validationItems = [] #Adding Ranger Plugin logic here
- ranger_plugin_properties = getSiteProperties(configurations, "ranger-hdfs-plugin-properties")
- ranger_plugin_enabled = ranger_plugin_properties['ranger-hdfs-plugin-enabled'] if ranger_plugin_properties else 'No'
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()):
- if 'dfs.namenode.inode.attributes.provider.class' not in hdfs_site or \
- hdfs_site['dfs.namenode.inode.attributes.provider.class'].lower() != 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer'.lower():
- validationItems.append({"config-name": 'dfs.namenode.inode.attributes.provider.class',
- "item": self.getWarnItem(
- "dfs.namenode.inode.attributes.provider.class needs to be set to 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer' if Ranger HDFS Plugin is enabled.")})
-
- # Check if dfs.allow.truncate is true
- if "HAWQ" in servicesList and \
- not ("dfs.allow.truncate" in services["configurations"]["hdfs-site"]["properties"] and \
- services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"].lower() == 'true'):
- validationItems.append({"config-name": "dfs.allow.truncate",
- "item": self.getWarnItem("HAWQ requires dfs.allow.truncate in hdfs-site.xml set to True.")})
-
- validationProblems = self.toConfigurationValidationProblems(validationItems, "hdfs-site")
- validationProblems.extend(parentValidationProblems)
- return validationProblems
-
-
- def validateHiveConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- parentValidationProblems = super(HDP23StackAdvisor, self).validateHiveConfigurations(properties, recommendedDefaults, configurations, services, hosts)
- hive_site = properties
- hive_env_properties = getSiteProperties(configurations, "hive-env")
- validationItems = []
- sqla_db_used = "hive_database" in hive_env_properties and \
- hive_env_properties['hive_database'] == 'Existing SQL Anywhere Database'
- prop_name = "datanucleus.rdbms.datastoreAdapterClassName"
- prop_value = "org.datanucleus.store.rdbms.adapter.SQLAnywhereAdapter"
- if sqla_db_used:
- if not prop_name in hive_site:
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem(
- "If Hive using SQL Anywhere db." \
- " {0} needs to be added with value {1}".format(prop_name,prop_value))})
- elif prop_name in hive_site and hive_site[prop_name] != "org.datanucleus.store.rdbms.adapter.SQLAnywhereAdapter":
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem(
- "If Hive using SQL Anywhere db." \
- " {0} needs to be set to {1}".format(prop_name,prop_value))})
-
- configurationValidationProblems = self.toConfigurationValidationProblems(validationItems, "hive-site")
- configurationValidationProblems.extend(parentValidationProblems)
- return configurationValidationProblems
-
- def validateHiveServer2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
- super(HDP23StackAdvisor, self).validateHiveServer2Configurations(properties, recommendedDefaults, configurations, services, hosts)
- hive_server2 = properties
- validationItems = []
- #Adding Ranger Plugin logic here
- ranger_plugin_properties = getSiteProperties(configurations, "ranger-hive-plugin-properties")
- hive_env_properties = getSiteProperties(configurations, "hive-env")
- ranger_plugin_enabled = 'hive_security_authorization' in hive_env_properties and hive_env_properties['hive_security_authorization'].lower() == 'ranger'
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- ##Add stack validations only if Ranger is enabled.
- if ("RANGER" in servicesList):
- ##Add stack validations for Ranger plugin enabled.
- if ranger_plugin_enabled:
- prop_name = 'hive.security.authorization.manager'
- prop_val = "org.apache.ranger.authorization.hive.authorizer.RangerHiveAuthorizerFactory"
- if prop_name in hive_server2 and hive_server2[prop_name] != prop_val:
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem(
- "If Ranger Hive Plugin is enabled."\
- " {0} under hiveserver2-site needs to be set to {1}".format(prop_name,prop_val))})
- prop_name = 'hive.security.authenticator.manager'
- prop_val = "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator"
- if prop_name in hive_server2 and hive_server2[prop_name] != prop_val:
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem(
- "If Ranger Hive Plugin is enabled."\
- " {0} under hiveserver2-site needs to be set to {1}".format(prop_name,prop_val))})
- prop_name = 'hive.security.authorization.enabled'
- prop_val = 'true'
- if prop_name in hive_server2 and hive_server2[prop_name] != prop_val:
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem(
- "If Ranger Hive Plugin is enabled."\
- " {0} under hiveserver2-site needs to be set to {1}".format(prop_name, prop_val))})
- prop_name = 'hive.conf.restricted.list'
- prop_vals = 'hive.security.authorization.enabled,hive.security.authorization.manager,hive.security.authenticator.manager'.split(',')
- current_vals = []
- missing_vals = []
- if hive_server2 and prop_name in hive_server2:
- current_vals = hive_server2[prop_name].split(',')
- current_vals = [x.strip() for x in current_vals]
-
- for val in prop_vals:
- if not val in current_vals:
- missing_vals.append(val)
-
- if missing_vals:
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem("If Ranger Hive Plugin is enabled."\
- " {0} under hiveserver2-site needs to contain missing value {1}".format(prop_name, ','.join(missing_vals)))})
- ##Add stack validations for Ranger plugin disabled.
- elif not ranger_plugin_enabled:
- prop_name = 'hive.security.authorization.manager'
- prop_val = "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"
- if prop_name in hive_server2 and hive_server2[prop_name] != prop_val:
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem(
- "If Ranger Hive Plugin is disabled."\
- " {0} needs to be set to {1}".format(prop_name,prop_val))})
- prop_name = 'hive.security.authenticator.manager'
- prop_val = "org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator"
- if prop_name in hive_server2 and hive_server2[prop_name] != prop_val:
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem(
- "If Ranger Hive Plugin is disabled."\
- " {0} needs to be set to {1}".format(prop_name,prop_val))})
- return self.toConfigurationValidationProblems(validationItems, "hiveserver2-site")
-
- def validateHBASEConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- super(HDP23StackAdvisor, self).validateHBASEConfigurations(properties, recommendedDefaults, configurations, services, hosts)
- hbase_site = properties
- validationItems = []
-
- #Adding Ranger Plugin logic here
- ranger_plugin_properties = getSiteProperties(configurations, "ranger-hbase-plugin-properties")
- ranger_plugin_enabled = ranger_plugin_properties['ranger-hbase-plugin-enabled'] if ranger_plugin_properties else 'No'
- prop_name = 'hbase.security.authorization'
- prop_val = "true"
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()):
- if hbase_site[prop_name] != prop_val:
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem(
- "If Ranger HBase Plugin is enabled."\
- "{0} needs to be set to {1}".format(prop_name,prop_val))})
- prop_name = "hbase.coprocessor.master.classes"
- prop_val = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor"
- exclude_val = "org.apache.hadoop.hbase.security.access.AccessController"
- if (prop_val in hbase_site[prop_name] and exclude_val not in hbase_site[prop_name]):
- pass
- else:
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem(
- "If Ranger HBase Plugin is enabled."\
- " {0} needs to contain {1} instead of {2}".format(prop_name,prop_val,exclude_val))})
- prop_name = "hbase.coprocessor.region.classes"
- prop_val = "org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor"
- if (prop_val in hbase_site[prop_name] and exclude_val not in hbase_site[prop_name]):
- pass
- else:
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem(
- "If Ranger HBase Plugin is enabled."\
- " {0} needs to contain {1} instead of {2}".format(prop_name,prop_val,exclude_val))})
-
- return self.toConfigurationValidationProblems(validationItems, "hbase-site")
-
- def validateKAFKAConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- kafka_broker = properties
- validationItems = []
-
- #Adding Ranger Plugin logic here
- ranger_plugin_properties = getSiteProperties(configurations, "ranger-kafka-plugin-properties")
- ranger_plugin_enabled = ranger_plugin_properties['ranger-kafka-plugin-enabled']
- prop_name = 'authorizer.class.name'
- prop_val = "org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer"
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- if ("RANGER" in servicesList) and (ranger_plugin_enabled.lower() == 'Yes'.lower()):
- if kafka_broker[prop_name] != prop_val:
- validationItems.append({"config-name": prop_name,
- "item": self.getWarnItem(
- "If Ranger Kafka Plugin is enabled."\
- "{0} needs to be set to {1}".format(prop_name,prop_val))})
-
- return self.toConfigurationValidationProblems(validationItems, "kafka-broker")
-
- def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- yarn_site = properties
- validationItems = []
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- if 'RANGER_KMS' in servicesList and 'KERBEROS' in servicesList:
- yarn_resource_proxy_enabled = yarn_site['yarn.resourcemanager.proxy-user-privileges.enabled']
- if yarn_resource_proxy_enabled.lower() == 'true':
- validationItems.append({"config-name": 'yarn.resourcemanager.proxy-user-privileges.enabled',
- "item": self.getWarnItem("If Ranger KMS service is installed set yarn.resourcemanager.proxy-user-privileges.enabled "\
- "property value as false under yarn-site"
- )})
-
- return self.toConfigurationValidationProblems(validationItems, "yarn-site")
-
- def isComponentUsingCardinalityForLayout(self, componentName):
- return componentName in ['NFS_GATEWAY', 'PHOENIX_QUERY_SERVER', 'SPARK_THRIFTSERVER']
-
- def validateRangerAdminConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- ranger_site = properties
- validationItems = []
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- if 'RANGER' in servicesList and 'policymgr_external_url' in ranger_site:
- policymgr_mgr_url = ranger_site['policymgr_external_url']
- if policymgr_mgr_url.endswith('/'):
- validationItems.append({'config-name':'policymgr_external_url',
- 'item':self.getWarnItem('Ranger External URL should not contain trailing slash "/"')})
- return self.toConfigurationValidationProblems(validationItems,'admin-properties')
-
+../../../HDP/2.3/services/stack_advisor.py
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b3f6d84/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/stack_advisor.py
index b8ea689..474e0c3 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.1/services/stack_advisor.py
@@ -60,4 +60,3 @@ class BigInsights41StackAdvisor(BigInsights40StackAdvisor):
componentHosts = [host for host in hosts["items"] if host["Hosts"]["host_name"] in componentHostnames]
return componentHosts
return []
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b3f6d84/ambari-server/src/main/resources/stacks/BigInsights/4.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/BigInsights/4.3/services/stack_advisor.py
index 4c43ecf..58e755a 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.3/services/stack_advisor.py
@@ -17,7 +17,59 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-class BigInsights43StackAdvisor(BigInsights42StackAdvisor):
-
- pass
-
\ No newline at end of file
+class BigInsights42StackAdvisor(BigInsights41StackAdvisor):
+
+ def getServiceConfigurationRecommenderDict(self):
+ parentRecommendConfDict = super(BigInsights42StackAdvisor, self).getServiceConfigurationRecommenderDict()
+ childRecommendConfDict = {
+ "YARN": self.recommendYARNConfigurations,
+ "KAFKA": self.recommendKAFKAConfigurations,
+ "SOLR": self.recommendSolrConfigurations
+ }
+ parentRecommendConfDict.update(childRecommendConfDict)
+ return parentRecommendConfDict
+
+ def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
+ super(BigInsights42StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts)
+ putYarnProperty = self.putProperty(configurations, "yarn-site", services)
+
+ # Property Attributes
+ putYarnPropertyAttribute = self.putPropertyAttribute(configurations, "yarn-site")
+ nodeManagerHost = self.getHostWithComponent("YARN", "NODEMANAGER", services, hosts)
+ if (nodeManagerHost is not None):
+ if "yarn-env" in services["configurations"] and "yarn_cgroups_enabled" in services["configurations"]["yarn-env"]["properties"]:
+ yarn_cgroups_enabled = services["configurations"]["yarn-env"]["properties"]["yarn_cgroups_enabled"].lower() == "true"
+ if yarn_cgroups_enabled:
+ putYarnProperty('yarn.nodemanager.container-executor.class', 'org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor')
+ putYarnProperty('yarn.nodemanager.linux-container-executor.group', 'hadoop')
+ putYarnProperty('yarn.nodemanager.linux-container-executor.resources-handler.class', 'org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler')
+ putYarnProperty('yarn.nodemanager.linux-container-executor.cgroups.hierarchy', '/yarn')
+ putYarnProperty('yarn.nodemanager.linux-container-executor.cgroups.mount', 'true')
+ putYarnProperty('yarn.nodemanager.linux-container-executor.cgroups.mount-path', '/cgroup')
+ else:
+ putYarnProperty('yarn.nodemanager.container-executor.class', 'org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor')
+ putYarnPropertyAttribute('yarn.nodemanager.linux-container-executor.resources-handler.class', 'delete', 'true')
+ putYarnPropertyAttribute('yarn.nodemanager.linux-container-executor.cgroups.hierarchy', 'delete', 'true')
+ putYarnPropertyAttribute('yarn.nodemanager.linux-container-executor.cgroups.mount', 'delete', 'true')
+ putYarnPropertyAttribute('yarn.nodemanager.linux-container-executor.cgroups.mount-path', 'delete', 'true')
+
+ if ("yarn-site" in services["configurations"]):
+ if ("yarn.nodemanager.resource.memory-mb" in services["configurations"]["yarn-site"]["properties"]):
+ # Compare the values derived from the hosts vs the yarn-site.xml static values, if the derived values are greater than the static ,
+ # override the static values with the derived to prevent the warnings that will appear in ambari
+ yarnPropertiesNmMemMb = int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
+ yarnConfigurationsNmMemMb = int(services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
+ if yarnPropertiesNmMemMb > yarnConfigurationsNmMemMb:
+ putYarnPropertyAttribute('yarn.scheduler.maximum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
+ putYarnPropertyAttribute('yarn.scheduler.minimum-allocation-mb', 'maximum', configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"])
+
+ def recommendKAFKAConfigurations(self, configurations, clusterData, services, hosts):
+ super(BigInsights42StackAdvisor, self).recommendKAFKAConfigurations(configurations, clusterData, services, hosts)
+
+ putKafkaBrokerAttributes = self.putPropertyAttribute(configurations, "kafka-broker")
+ putKafkaBrokerAttributes('port','delete','true')
+
+ def recommendSolrConfigurations(self, configurations, clusterData, services, hosts):
+ putSolrEnvProperty = self.putProperty(configurations, "solr-env", services)
+ putSolrAttributes = self.putPropertyAttribute(configurations, "solr-env")
+ putSolrAttributes('solr_lib_dir','delete','true')
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b3f6d84/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 9f74dc5..34b4614 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -17,8 +17,6 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-from stack_advisor_206 import *
-
class HDP21StackAdvisor(HDP206StackAdvisor):
def getServiceConfigurationRecommenderDict(self):
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b3f6d84/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 65ec715..4854514 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -26,8 +26,6 @@ import socket
import re
import xml.etree.ElementTree as ET
-from stack_advisor_21 import *
-
class HDP22StackAdvisor(HDP21StackAdvisor):
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b3f6d84/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index 28cece6..a0cce7b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -24,23 +24,17 @@ import fnmatch
import math
import socket
-from stack_advisor_22 import *
+# Local Imports
+from resource_management.core.logger import Logger
-class HDP23StackAdvisor(HDP22StackAdvisor):
- def createComponentLayoutRecommendations(self, services, hosts):
- parentComponentLayoutRecommendations = super(HDP23StackAdvisor, self).createComponentLayoutRecommendations(services, hosts)
+DB_TYPE_DEFAULT_PORT_MAP = {"MYSQL":"3306", "ORACLE":"1521", "POSTGRES":"5432", "MSSQL":"1433", "SQLA":"2638"}
- # remove HAWQSTANDBY on a single node
- hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
- if len(hostsList) == 1:
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- if "HAWQ" in servicesList:
- components = parentComponentLayoutRecommendations["blueprint"]["host_groups"][0]["components"]
- components = [ component for component in components if component["name"] != 'HAWQSTANDBY' ]
- parentComponentLayoutRecommendations["blueprint"]["host_groups"][0]["components"] = components
+class HDP23StackAdvisor(HDP22StackAdvisor):
- return parentComponentLayoutRecommendations
+ def __init__(self):
+ super(HDP23StackAdvisor, self).__init__()
+ Logger.initialize_logger()
def getComponentLayoutValidations(self, services, hosts):
parentItems = super(HDP23StackAdvisor, self).getComponentLayoutValidations(services, hosts)