You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2016/11/17 17:13:28 UTC
[12/50] [abbrv] ambari git commit: AMBARI-18818 Config changes for
Atlas to Support KnoxSSO Authentication (mugdha)
AMBARI-18818 Config changes for Atlas to Support KnoxSSO Authentication (mugdha)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5454bd61
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5454bd61
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5454bd61
Branch: refs/heads/branch-feature-AMBARI-18456
Commit: 5454bd612774d7f17704adaff24cbfa73c6a4f77
Parents: 8c3ad1d
Author: Mugdha Varadkar <mu...@apache.org>
Authored: Tue Nov 15 11:30:25 2016 +0530
Committer: Mugdha Varadkar <mu...@apache.org>
Committed: Wed Nov 16 10:23:32 2016 +0530
----------------------------------------------------------------------
.../configuration/application-properties.xml | 11 +
.../configuration/application-properties.xml | 95 +++
.../ATLAS/configuration/atlas-log4j.xml | 118 +++
.../stacks/HDP/2.6/services/ATLAS/metainfo.xml | 10 +-
.../services/ATLAS/themes/theme_version_2.json | 845 +++++++++++++++++++
.../stacks/HDP/2.6/services/stack_advisor.py | 381 +++++----
.../stacks/2.6/common/test_stack_advisor.py | 97 +++
7 files changed, 1374 insertions(+), 183 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
index 95715b3..fda16b2 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/configuration/application-properties.xml
@@ -77,18 +77,29 @@
<value>false</value>
<description>Indicates whether or not Kerberos is enabled.</description>
<on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
</property>
<property>
<name>atlas.authentication.method.file</name>
+ <display-name>Enable File Authentication</display-name>
<value>true</value>
<description>Indicates whether or not file based authentication is enabled.</description>
<on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
</property>
<property>
<name>atlas.authentication.method.ldap</name>
+ <display-name>Enable LDAP Authentication</display-name>
<value>false</value>
<description>Indicates whether or not LDAP authentication is enabled.</description>
<on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
</property>
<property>
<name>atlas.authentication.method.file.filename</name>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml
new file mode 100644
index 0000000..0d6ee38
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/application-properties.xml
@@ -0,0 +1,95 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false">
+
+ <!-- Atlas knox sso properties -->
+ <property>
+ <name>atlas.sso.knox.enabled</name>
+ <display-name>Enable Atlas Knox SSO</display-name>
+ <value>false</value>
+ <description/>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>atlas.sso.knox.providerurl</name>
+ <value/>
+ <description/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>gateway-site</type>
+ <name>gateway.port</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>atlas.sso.knox.publicKey</name>
+ <value/>
+ <description/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>atlas.sso.knox.browser.useragent</name>
+ <value/>
+ <description/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>atlas.authentication.method.ldap.type</name>
+ <display-name>LDAP Authentication Type</display-name>
+ <value>ldap</value>
+ <description>The LDAP type (ldap, ad, or none).</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>ldap</value>
+ <label>LDAP</label>
+ </entry>
+ <entry>
+ <value>ad</value>
+ <label>AD</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/atlas-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/atlas-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/atlas-log4j.xml
new file mode 100644
index 0000000..aebf241
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/configuration/atlas-log4j.xml
@@ -0,0 +1,118 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <property>
+ <name>content</name>
+ <display-name>atlas-log4j template</display-name>
+ <description>Custom log4j.properties</description>
+ <value><![CDATA[<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+ <appender name="console" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
+ </layout>
+ </appender>
+
+ <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="{{log_dir}}/application.log"/>
+ <param name="Append" value="true"/>
+ <param name="Threshold" value="info"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n"/>
+ </layout>
+ </appender>
+
+ <appender name="AUDIT" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="{{log_dir}}/audit.log"/>
+ <param name="Append" value="true"/>
+ <param name="Threshold" value="info"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %x %m%n"/>
+ </layout>
+ </appender>
+
+ <logger name="org.apache.atlas" additivity="false">
+ <level value="{{atlas_log_level}}"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+
+ <logger name="com.thinkaurelius.titan" additivity="false">
+ <level value="info"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="org.elasticsearch" additivity="false">
+ <level value="info"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="org.apache.lucene" additivity="false">
+ <level value="info"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="com.google" additivity="false">
+ <level value="info"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="AUDIT" additivity="false">
+ <level value="{{audit_log_level}}"/>
+ <appender-ref ref="AUDIT"/>
+ </logger>
+
+ <root>
+ <priority value="info"/>
+ <appender-ref ref="FILE"/>
+ </root>
+
+</log4j:configuration>
+ ]]></value>
+ <value-attributes>
+ <type>content</type>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/metainfo.xml
index 92db1f9..eaa5e21 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/metainfo.xml
@@ -20,7 +20,15 @@
<services>
<service>
<name>ATLAS</name>
- <version>0.7.0.2.5</version>
+ <version>0.8.0.2.6</version>
+
+ <themes>
+ <theme>
+ <fileName>theme_version_2.json</fileName>
+ <default>true</default>
+ </theme>
+ </themes>
+
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/themes/theme_version_2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/themes/theme_version_2.json b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/themes/theme_version_2.json
new file mode 100644
index 0000000..f72ea4b
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/ATLAS/themes/theme_version_2.json
@@ -0,0 +1,845 @@
+{
+ "name": "default",
+ "description": "Default theme for Atlas service",
+ "configuration": {
+ "layouts": [
+ {
+ "name": "default",
+ "tabs": [
+ {
+ "name": "authentication_settings",
+ "display-name": "Authentication",
+ "layout": {
+ "tab-columns": "1",
+ "tab-rows": "4",
+ "sections": [
+ {
+ "name": "section-authentication-type",
+ "display-name": "Authentication Methods",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-authentication-type",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ },
+ {
+ "name": "subsection-authentication-file",
+ "display-name": "File",
+ "row-index": "1",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-authentication-file",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ },
+ {
+ "name": "section-authentication-ldap",
+ "display-name": "LDAP/AD",
+ "row-index": "2",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-authentication-ldap",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "section-atlas-sso",
+ "display-name": "Atlas Knox SSO",
+ "row-index": "3",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1",
+ "section-columns": "1",
+ "section-rows": "1",
+ "subsections": [
+ {
+ "name": "subsection-atlas-sso",
+ "row-index": "0",
+ "column-index": "0",
+ "row-span": "1",
+ "column-span": "1"
+ }
+ ]
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "placement": {
+ "configuration-layout": "default",
+ "configs": [
+ {
+ "config": "application-properties/atlas.authentication.method.file",
+ "subsection-name": "subsection-authentication-type"
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.file.filename",
+ "subsection-name": "subsection-authentication-file",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.file"
+ ],
+ "if": "${application-properties/atlas.authentication.method.file}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap",
+ "subsection-name": "subsection-authentication-type"
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.type",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.url",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.userDNpattern",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap} ",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.groupSearchBase",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.groupSearchFilter",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.groupRoleAttribute",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.base.dn",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.bind.dn",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.bind.password",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.referral",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.user.searchfilter",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.default.role",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ldap && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.url",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.domain",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.base.dn",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.bind.dn",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.bind.password",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.referral",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.user.searchfilter",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.default.role",
+ "subsection-name": "subsection-authentication-ldap",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.authentication.method.ldap.type",
+ "application-properties/atlas.authentication.method.ldap"
+ ],
+ "if": "${application-properties/atlas.authentication.method.ldap.type} === ad && ${application-properties/atlas.authentication.method.ldap}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.sso.enabled",
+ "subsection-name": "subsection-authentication-type"
+ },
+ {
+ "config": "application-properties/atlas.sso.providerurl",
+ "subsection-name": "subsection-atlas-sso",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.sso.enabled"
+ ],
+ "if": "${application-properties/atlas.sso.enabled}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.sso.publicKey",
+ "subsection-name": "subsection-atlas-sso",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.sso.enabled"
+ ],
+ "if": "${application-properties/atlas.sso.enabled}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ },
+ {
+ "config": "application-properties/atlas.sso.browser.useragent",
+ "subsection-name": "subsection-atlas-sso",
+ "depends-on": [
+ {
+ "configs":[
+ "application-properties/atlas.sso.enabled"
+ ],
+ "if": "${application-properties/atlas.sso.enabled}",
+ "then": {
+ "property_value_attributes": {
+ "visible": true
+ }
+ },
+ "else": {
+ "property_value_attributes": {
+ "visible": false
+ }
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "widgets": [
+ {
+ "config": "application-properties/atlas.authentication.method.file",
+ "widget": {
+ "type": "checkbox"
+ }
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.file.filename",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap",
+ "widget": {
+ "type": "checkbox"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.url",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.userDNpattern",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.groupSearchBase",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.groupSearchFilter",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.groupRoleAttribute",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.base.dn",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.bind.dn",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.bind.password",
+ "widget":{
+ "type":"password"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.referral",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.user.searchfilter",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.default.role",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.type",
+ "widget": {
+ "type": "combo"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.ad.url",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.ad.domain",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config":"application-properties/atlas.authentication.method.ldap.ad.base.dn",
+ "widget":{
+ "type":"text-field"
+ }
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.bind.dn",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.bind.password",
+ "widget": {
+ "type": "password"
+ }
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.referral",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.user.searchfilter",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "application-properties/atlas.authentication.method.ldap.ad.default.role",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "application-properties/atlas.sso.enabled",
+ "widget": {
+ "type":"checkbox"
+ }
+ },
+ {
+ "config": "application-properties/atlas.sso.providerurl",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "application-properties/atlas.sso.publicKey",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
+ "config": "application-properties/atlas.sso.browser.useragent",
+ "widget": {
+ "type": "text-field"
+ }
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 49dd086..718da2c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -22,185 +22,202 @@ from resource_management.libraries.functions import format
class HDP26StackAdvisor(HDP25StackAdvisor):
- def __init__(self):
- super(HDP26StackAdvisor, self).__init__()
- Logger.initialize_logger()
-
- def getServiceConfigurationRecommenderDict(self):
- parentRecommendConfDict = super(HDP26StackAdvisor, self).getServiceConfigurationRecommenderDict()
- childRecommendConfDict = {
- "DRUID": self.recommendDruidConfigurations
- }
- parentRecommendConfDict.update(childRecommendConfDict)
- return parentRecommendConfDict
-
- def recommendDruidConfigurations(self, configurations, clusterData, services, hosts):
-
- componentsListList = [service["components"] for service in services["services"]]
- componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
- servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
- putCommonProperty = self.putProperty(configurations, "druid-common", services)
-
- putCommonProperty('druid.zk.service.host', self.getZKHostPortString(services))
- self.recommendDruidMaxMemoryLimitConfigurations(configurations, clusterData, services, hosts)
-
- # recommending the metadata storage uri
- database_name = services['configurations']["druid-common"]["properties"]["database_name"]
- metastore_hostname = services['configurations']["druid-common"]["properties"]["metastore_hostname"]
- database_type = services['configurations']["druid-common"]["properties"]["druid.metadata.storage.type"]
- metadata_storage_port = "1527"
- mysql_extension_name = "io.druid.extensions:mysql-metadata-storage"
- mysql_module_name = "mysql-metadata-storage"
- postgres_module_name = "postgresql-metadata-storage"
- extensions_load_list = services['configurations']['druid-common']['properties']['druid.extensions.loadList']
- extensions_pull_list = services['configurations']['druid-common']['properties']['druid.extensions.pullList']
- putDruidCommonProperty = self.putProperty(configurations, "druid-common", services)
-
- extensions_pull_list = self.removeFromList(extensions_pull_list, mysql_extension_name)
- extensions_load_list = self.removeFromList(extensions_load_list, mysql_module_name)
- extensions_load_list = self.removeFromList(extensions_load_list, postgres_module_name)
-
- if database_type == 'mysql':
- metadata_storage_port = "3306"
- extensions_pull_list = self.addToList(extensions_pull_list, mysql_extension_name)
- extensions_load_list = self.addToList(extensions_load_list, mysql_module_name)
-
- if database_type == 'postgres':
- extensions_load_list = self.addToList(extensions_load_list, postgres_module_name)
- metadata_storage_port = "5432"
-
- putDruidCommonProperty('druid.metadata.storage.connector.port', metadata_storage_port)
- putDruidCommonProperty('druid.metadata.storage.connector.connectURI',
- self.getMetadataConnectionString(database_type).format(metastore_hostname, database_name,
- metadata_storage_port))
- # HDFS is installed
- if "HDFS" in servicesList and "hdfs-site" in services["configurations"]:
- # recommend HDFS as default deep storage
- extensions_load_list = self.addToList(extensions_load_list, "druid-hdfs-storage")
- putCommonProperty("druid.storage.type", "hdfs")
- putCommonProperty("druid.storage.storageDirectory", "/user/druid/data")
- # configure indexer logs configs
- putCommonProperty("druid.indexer.logs.type", "hdfs")
- putCommonProperty("druid.indexer.logs.directory", "/user/druid/logs")
-
- if "KAFKA" in servicesList:
- extensions_load_list = self.addToList(extensions_load_list, "druid-kafka-indexing-service")
-
- putCommonProperty('druid.extensions.loadList', extensions_load_list)
- putCommonProperty('druid.extensions.pullList', extensions_pull_list)
-
- # JVM Configs go to env properties
- putEnvProperty = self.putProperty(configurations, "druid-env", services)
-
- # processing thread pool Config
- for component in ['DRUID_HISTORICAL', 'DRUID_BROKER']:
- component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts)
- nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component]
- putComponentProperty = self.putProperty(configurations, format("druid-{nodeType}"), services)
- if (component_hosts is not None and len(component_hosts) > 0):
- totalAvailableCpu = self.getMinCpu(component_hosts)
- processingThreads = 1
- if totalAvailableCpu > 1:
- processingThreads = totalAvailableCpu - 1
- putComponentProperty('druid.processing.numThreads', processingThreads)
- putComponentProperty('druid.server.http.numThreads', max(10, (totalAvailableCpu * 17) / 16 + 2) + 30)
-
- def getMetadataConnectionString(self, database_type):
- driverDict = {
- 'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true',
- 'derby': 'jdbc:derby://{0}:{2}/{1};create=true',
- 'postgres': 'jdbc:postgresql://{0}:{2}/{1}'
- }
- return driverDict.get(database_type.lower())
-
- def addToList(self, json_list, word):
- desr_list = json.loads(json_list)
- if word not in desr_list:
- desr_list.append(word)
- return json.dumps(desr_list)
-
- def removeFromList(self, json_list, word):
- desr_list = json.loads(json_list)
- if word in desr_list:
- desr_list.remove(word)
- return json.dumps(desr_list)
-
- def recommendDruidMaxMemoryLimitConfigurations(self, configurations, clusterData, services, hosts):
- putEnvPropertyAttribute = self.putPropertyAttribute(configurations, "druid-env")
- for component in ["DRUID_HISTORICAL", "DRUID_MIDDLEMANAGER", "DRUID_BROKER", "DRUID_OVERLORD",
- "DRUID_COORDINATOR"]:
- component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts)
- if component_hosts is not None and len(component_hosts) > 0:
- totalAvailableMem = self.getMinMemory(component_hosts) / 1024 # In MB
- nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component]
- putEnvPropertyAttribute(format('druid.{nodeType}.jvm.heap.memory'), 'maximum',
- max(totalAvailableMem, 1024))
-
- DRUID_COMPONENT_NODE_TYPE_MAP = {
- 'DRUID_BROKER': 'broker',
- 'DRUID_COORDINATOR': 'coordinator',
- 'DRUID_HISTORICAL': 'historical',
- 'DRUID_MIDDLEMANAGER': 'middlemanager',
- 'DRUID_OVERLORD': 'overlord',
- 'DRUID_ROUTER': 'router'
- }
-
- def getMinMemory(self, component_hosts):
- min_ram_kb = 1073741824 # 1 TB
- for host in component_hosts:
- ram_kb = host['Hosts']['total_mem']
- min_ram_kb = min(min_ram_kb, ram_kb)
- return min_ram_kb
-
- def getMinCpu(self, component_hosts):
- min_cpu = 256
- for host in component_hosts:
- cpu_count = host['Hosts']['cpu_count']
- min_cpu = min(min_cpu, cpu_count)
- return min_cpu
-
- def getServiceConfigurationValidators(self):
- parentValidators = super(HDP26StackAdvisor, self).getServiceConfigurationValidators()
- childValidators = {
- "DRUID": {"druid-env": self.validateDruidEnvConfigurations,
- "druid-historical": self.validateDruidHistoricalConfigurations,
- "druid-broker": self.validateDruidBrokerConfigurations}
- }
- self.mergeValidators(parentValidators, childValidators)
- return parentValidators
-
- def validateDruidEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- validationItems = []
- # Minimum Direct memory Validation
- envProperties = services['configurations']['druid-env']['properties']
- for nodeType in ['broker', 'historical']:
- properties = services['configurations'][format('druid-{nodeType}')]['properties']
- intermediateBufferSize = int(properties['druid.processing.buffer.sizeBytes']) / (1024 * 1024) # In MBs
- processingThreads = int(properties['druid.processing.numThreads'])
- directMemory = int(envProperties[format('druid.{nodeType}.jvm.direct.memory')])
- if directMemory < (processingThreads + 1) * intermediateBufferSize:
- validationItems.extend(
- {"config-name": format("druid.{nodeType}.jvm.direct.memory"), "item": self.getErrorItem(
- format(
- "Not enough direct memory available for {nodeType} Node."
- "Please adjust druid.{nodeType}.jvm.direct.memory, druid.processing.buffer.sizeBytes, druid.processing.numThreads"
- )
- )
- })
- return self.toConfigurationValidationProblems(validationItems, "druid-env")
-
- def validateDruidHistoricalConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- validationItems = [
- {"config-name": "druid.processing.numThreads",
- "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults,
- "druid.processing.numThreads")}
- ]
- return self.toConfigurationValidationProblems(validationItems, "druid-historical")
-
- def validateDruidBrokerConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- validationItems = [
- {"config-name": "druid.processing.numThreads",
- "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults,
- "druid.processing.numThreads")}
- ]
- return self.toConfigurationValidationProblems(validationItems, "druid-broker")
+ def __init__(self):
+ super(HDP26StackAdvisor, self).__init__()
+ Logger.initialize_logger()
+
+ def getServiceConfigurationRecommenderDict(self):
+ parentRecommendConfDict = super(HDP26StackAdvisor, self).getServiceConfigurationRecommenderDict()
+ childRecommendConfDict = {
+ "DRUID": self.recommendDruidConfigurations,
+ "ATLAS": self.recommendAtlasConfigurations
+ }
+ parentRecommendConfDict.update(childRecommendConfDict)
+ return parentRecommendConfDict
+
+ def recommendAtlasConfigurations(self, configurations, clusterData, services, hosts):
+ super(HDP26StackAdvisor, self).recommendAtlasConfigurations(configurations, clusterData, services, hosts)
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ putAtlasApplicationProperty = self.putProperty(configurations, "application-properties", services)
+
+ knox_host = 'localhost'
+ knox_port = '8443'
+ if 'KNOX' in servicesList:
+ knox_hosts = self.getComponentHostNames(services, "KNOX", "KNOX_GATEWAY")
+ if len(knox_hosts) > 0:
+ knox_hosts.sort()
+ knox_host = knox_hosts[0]
+ if 'gateway-site' in services['configurations'] and 'gateway.port' in services['configurations']["gateway-site"]["properties"]:
+ knox_port = services['configurations']["gateway-site"]["properties"]['gateway.port']
+ putAtlasApplicationProperty('atlas.sso.providerurl', 'https://{0}:{1}/gateway/knoxsso/api/v1/websso'.format(knox_host, knox_port))
+
+ def recommendDruidConfigurations(self, configurations, clusterData, services, hosts):
+
+ componentsListList = [service["components"] for service in services["services"]]
+ componentsList = [item["StackServiceComponents"] for sublist in componentsListList for item in sublist]
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+ putCommonProperty = self.putProperty(configurations, "druid-common", services)
+
+ putCommonProperty('druid.zk.service.host', self.getZKHostPortString(services))
+ self.recommendDruidMaxMemoryLimitConfigurations(configurations, clusterData, services, hosts)
+
+ # recommending the metadata storage uri
+ database_name = services['configurations']["druid-common"]["properties"]["database_name"]
+ metastore_hostname = services['configurations']["druid-common"]["properties"]["metastore_hostname"]
+ database_type = services['configurations']["druid-common"]["properties"]["druid.metadata.storage.type"]
+ metadata_storage_port = "1527"
+ mysql_extension_name = "io.druid.extensions:mysql-metadata-storage"
+ mysql_module_name = "mysql-metadata-storage"
+ postgres_module_name = "postgresql-metadata-storage"
+ extensions_load_list = services['configurations']['druid-common']['properties']['druid.extensions.loadList']
+ extensions_pull_list = services['configurations']['druid-common']['properties']['druid.extensions.pullList']
+ putDruidCommonProperty = self.putProperty(configurations, "druid-common", services)
+
+ extensions_pull_list = self.removeFromList(extensions_pull_list, mysql_extension_name)
+ extensions_load_list = self.removeFromList(extensions_load_list, mysql_module_name)
+ extensions_load_list = self.removeFromList(extensions_load_list, postgres_module_name)
+
+ if database_type == 'mysql':
+ metadata_storage_port = "3306"
+ extensions_pull_list = self.addToList(extensions_pull_list, mysql_extension_name)
+ extensions_load_list = self.addToList(extensions_load_list, mysql_module_name)
+
+ if database_type == 'postgres':
+ extensions_load_list = self.addToList(extensions_load_list, postgres_module_name)
+ metadata_storage_port = "5432"
+
+ putDruidCommonProperty('druid.metadata.storage.connector.port', metadata_storage_port)
+ putDruidCommonProperty('druid.metadata.storage.connector.connectURI',
+ self.getMetadataConnectionString(database_type).format(metastore_hostname, database_name,
+ metadata_storage_port))
+ # HDFS is installed
+ if "HDFS" in servicesList and "hdfs-site" in services["configurations"]:
+ # recommend HDFS as default deep storage
+ extensions_load_list = self.addToList(extensions_load_list, "druid-hdfs-storage")
+ putCommonProperty("druid.storage.type", "hdfs")
+ putCommonProperty("druid.storage.storageDirectory", "/user/druid/data")
+ # configure indexer logs configs
+ putCommonProperty("druid.indexer.logs.type", "hdfs")
+ putCommonProperty("druid.indexer.logs.directory", "/user/druid/logs")
+
+ if "KAFKA" in servicesList:
+ extensions_load_list = self.addToList(extensions_load_list, "druid-kafka-indexing-service")
+
+ putCommonProperty('druid.extensions.loadList', extensions_load_list)
+ putCommonProperty('druid.extensions.pullList', extensions_pull_list)
+
+ # JVM Configs go to env properties
+ putEnvProperty = self.putProperty(configurations, "druid-env", services)
+
+ # processing thread pool Config
+ for component in ['DRUID_HISTORICAL', 'DRUID_BROKER']:
+ component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts)
+ nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component]
+ putComponentProperty = self.putProperty(configurations, format("druid-{nodeType}"), services)
+ if (component_hosts is not None and len(component_hosts) > 0):
+ totalAvailableCpu = self.getMinCpu(component_hosts)
+ processingThreads = 1
+ if totalAvailableCpu > 1:
+ processingThreads = totalAvailableCpu - 1
+ putComponentProperty('druid.processing.numThreads', processingThreads)
+ putComponentProperty('druid.server.http.numThreads', max(10, (totalAvailableCpu * 17) / 16 + 2) + 30)
+
+ def getMetadataConnectionString(self, database_type):
+ driverDict = {
+ 'mysql': 'jdbc:mysql://{0}:{2}/{1}?createDatabaseIfNotExist=true',
+ 'derby': 'jdbc:derby://{0}:{2}/{1};create=true',
+ 'postgres': 'jdbc:postgresql://{0}:{2}/{1}'
+ }
+ return driverDict.get(database_type.lower())
+
+ def addToList(self, json_list, word):
+ desr_list = json.loads(json_list)
+ if word not in desr_list:
+ desr_list.append(word)
+ return json.dumps(desr_list)
+
+ def removeFromList(self, json_list, word):
+ desr_list = json.loads(json_list)
+ if word in desr_list:
+ desr_list.remove(word)
+ return json.dumps(desr_list)
+
+ def recommendDruidMaxMemoryLimitConfigurations(self, configurations, clusterData, services, hosts):
+ putEnvPropertyAttribute = self.putPropertyAttribute(configurations, "druid-env")
+ for component in ["DRUID_HISTORICAL", "DRUID_MIDDLEMANAGER", "DRUID_BROKER", "DRUID_OVERLORD",
+ "DRUID_COORDINATOR"]:
+ component_hosts = self.getHostsWithComponent("DRUID", component, services, hosts)
+ if component_hosts is not None and len(component_hosts) > 0:
+ totalAvailableMem = self.getMinMemory(component_hosts) / 1024 # In MB
+ nodeType = self.DRUID_COMPONENT_NODE_TYPE_MAP[component]
+ putEnvPropertyAttribute(format('druid.{nodeType}.jvm.heap.memory'), 'maximum',
+ max(totalAvailableMem, 1024))
+
+ DRUID_COMPONENT_NODE_TYPE_MAP = {
+ 'DRUID_BROKER': 'broker',
+ 'DRUID_COORDINATOR': 'coordinator',
+ 'DRUID_HISTORICAL': 'historical',
+ 'DRUID_MIDDLEMANAGER': 'middlemanager',
+ 'DRUID_OVERLORD': 'overlord',
+ 'DRUID_ROUTER': 'router'
+ }
+
+ def getMinMemory(self, component_hosts):
+ min_ram_kb = 1073741824 # 1 TB
+ for host in component_hosts:
+ ram_kb = host['Hosts']['total_mem']
+ min_ram_kb = min(min_ram_kb, ram_kb)
+ return min_ram_kb
+
+ def getMinCpu(self, component_hosts):
+ min_cpu = 256
+ for host in component_hosts:
+ cpu_count = host['Hosts']['cpu_count']
+ min_cpu = min(min_cpu, cpu_count)
+ return min_cpu
+
+ def getServiceConfigurationValidators(self):
+ parentValidators = super(HDP26StackAdvisor, self).getServiceConfigurationValidators()
+ childValidators = {
+ "DRUID": {"druid-env": self.validateDruidEnvConfigurations,
+ "druid-historical": self.validateDruidHistoricalConfigurations,
+ "druid-broker": self.validateDruidBrokerConfigurations}
+ }
+ self.mergeValidators(parentValidators, childValidators)
+ return parentValidators
+
+ def validateDruidEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = []
+ # Minimum Direct memory Validation
+ envProperties = services['configurations']['druid-env']['properties']
+ for nodeType in ['broker', 'historical']:
+ properties = services['configurations'][format('druid-{nodeType}')]['properties']
+ intermediateBufferSize = int(properties['druid.processing.buffer.sizeBytes']) / (1024 * 1024) # In MBs
+ processingThreads = int(properties['druid.processing.numThreads'])
+ directMemory = int(envProperties[format('druid.{nodeType}.jvm.direct.memory')])
+ if directMemory < (processingThreads + 1) * intermediateBufferSize:
+ validationItems.extend(
+ {"config-name": format("druid.{nodeType}.jvm.direct.memory"), "item": self.getErrorItem(
+ format(
+ "Not enough direct memory available for {nodeType} Node."
+ "Please adjust druid.{nodeType}.jvm.direct.memory, druid.processing.buffer.sizeBytes, druid.processing.numThreads"
+ )
+ )
+ })
+ return self.toConfigurationValidationProblems(validationItems, "druid-env")
+
+ def validateDruidHistoricalConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = [
+ {"config-name": "druid.processing.numThreads",
+ "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults,
+ "druid.processing.numThreads")}
+ ]
+ return self.toConfigurationValidationProblems(validationItems, "druid-historical")
+
+ def validateDruidBrokerConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+ validationItems = [
+ {"config-name": "druid.processing.numThreads",
+ "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults,
+ "druid.processing.numThreads")}
+ ]
+ return self.toConfigurationValidationProblems(validationItems, "druid-broker")
http://git-wip-us.apache.org/repos/asf/ambari/blob/5454bd61/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
index 8227d69..a7a45f8 100644
--- a/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.6/common/test_stack_advisor.py
@@ -568,6 +568,103 @@ class TestHDP26StackAdvisor(TestCase):
)
+ def test_recommendAtlasConfigurations(self):
+ configurations = {
+ "application-properties": {
+ "properties": {
+ "atlas.sso.providerurl": "",
+ "atlas.graph.index.search.solr.zookeeper-url": "",
+ "atlas.audit.hbase.zookeeper.quorum": "",
+ "atlas.graph.storage.hostname": "",
+ "atlas.kafka.bootstrap.servers": "",
+ "atlas.kafka.zookeeper.connect": "",
+ "atlas.authorizer.impl": "simple"
+ }
+ },
+ "infra-solr-env": {
+ "properties": {
+ "infra_solr_znode": "/infra-solr"
+ }
+ },
+ "ranger-atlas-plugin-properties": {
+ "properties": {
+ "ranger-atlas-plugin-enabled":"No"
+ }
+ },
+ "atlas-env": {
+ "properties": {
+ "atlas_server_max_new_size": "600",
+ "atlas_server_xmx": "2048"
+ }
+ }
+ }
+
+ clusterData = {}
+
+ expected = {
+ "application-properties": {
+ "properties": {
+ "atlas.sso.providerurl": "https://c6401.ambari.apache.org:8443/gateway/knoxsso/api/v1/websso",
+ "atlas.graph.index.search.solr.zookeeper-url": "",
+ "atlas.audit.hbase.zookeeper.quorum": "",
+ "atlas.graph.storage.hostname": "",
+ "atlas.kafka.bootstrap.servers": "",
+ "atlas.kafka.zookeeper.connect": "",
+ "atlas.authorizer.impl": "simple"
+ }
+ },
+ "infra-solr-env": {
+ "properties": {
+ "infra_solr_znode": "/infra-solr"
+ }
+ },
+ "ranger-atlas-plugin-properties": {
+ "properties": {
+ "ranger-atlas-plugin-enabled":"No"
+ }
+ },
+ "atlas-env": {
+ "properties": {
+ "atlas_server_max_new_size": "600",
+ "atlas_server_xmx": "2048"
+ }
+ }
+ }
+
+ services = {
+ "services": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.6/services/KNOX",
+ "StackServices": {
+ "service_name": "KNOX",
+ "service_version": "0.9.0.2.5",
+ "stack_name": "HDP",
+ "stack_version": "2.6"
+ },
+ "components": [
+ {
+ "href": "/api/v1/stacks/HDP/versions/2.6/services/KNOX/components/KNOX_GATEWAY",
+ "StackServiceComponents": {
+ "advertise_version": "false",
+ "cardinality": "1+",
+ "component_category": "MASTER",
+ "component_name": "KNOX_GATEWAY",
+ "display_name": "Knox Gateway",
+ "is_client": "false",
+ "is_master": "true",
+ "hostnames": ["c6401.ambari.apache.org"]
+ },
+ "dependencies": []
+ }
+ ]
+ }
+ ],
+ "configurations": configurations
+ }
+
+ self.stackAdvisor.recommendAtlasConfigurations(configurations, clusterData, services, None)
+ self.assertEquals(configurations, expected)
+
def load_json(self, filename):
file = os.path.join(self.testDirectory, filename)
with open(file, 'rb') as f: