You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mu...@apache.org on 2018/07/27 04:54:27 UTC
[ambari] branch branch-2.7 updated: AMBARI-24352 Updating Ranger
and Ranger KMS configs during Ambari upgrade (mugdha) (#1878)
This is an automated email from the ASF dual-hosted git repository.
mugdha pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new d930ad4 AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade (mugdha) (#1878)
d930ad4 is described below
commit d930ad4faa5898d992fb0055e86a261d31195924
Author: Mugdha Varadkar <fi...@users.noreply.github.com>
AuthorDate: Fri Jul 27 10:24:25 2018 +0530
AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade (mugdha) (#1878)
* AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade (mugdha)
* AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade. Minor UT fix. (mugdha)
* AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade. Adding Doc Comment. (mugdha)
* AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade. Adding logic for fetching port from existing db conn string. (mugdha)
---
.../ambari/server/upgrade/SchemaUpgradeHelper.java | 1 +
.../ambari/server/upgrade/UpgradeCatalog271.java | 198 ++++++++++++++++++
.../KAFKA/0.8.1/package/scripts/params.py | 2 -
.../RANGER/0.4.0/package/scripts/params.py | 6 +-
.../package/templates/input.config-ranger.json.j2 | 6 +-
.../0.5.0/configuration/ranger-ugsync-site.xml | 4 +-
.../0.7.0/configuration/ranger-admin-site.xml | 11 +
.../RANGER/0.7.0/configuration/ranger-env.xml | 10 +
.../RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml | 20 ++
.../YARN/2.1.0.2.0/package/scripts/params_linux.py | 2 -
.../RANGER_KMS/themes/theme_version_1.json | 49 ++++-
.../stacks/HDP/2.3/services/stack_advisor.py | 19 ++
.../server/upgrade/UpgradeCatalog271Test.java | 229 +++++++++++++++++++++
.../python/stacks/2.3/common/test_stack_advisor.py | 10 +
.../stacks/2.5/configs/ranger-admin-default.json | 2 +-
.../stacks/2.5/configs/ranger-admin-secured.json | 2 +-
.../stacks/2.6/configs/ranger-admin-default.json | 2 +-
.../stacks/2.6/configs/ranger-admin-secured.json | 2 +-
18 files changed, 552 insertions(+), 23 deletions(-)
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
index de80052..4a05175 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java
@@ -189,6 +189,7 @@ public class SchemaUpgradeHelper {
catalogBinder.addBinding().to(UpgradeCatalog261.class);
catalogBinder.addBinding().to(UpgradeCatalog262.class);
catalogBinder.addBinding().to(UpgradeCatalog270.class);
+ catalogBinder.addBinding().to(UpgradeCatalog271.class);
catalogBinder.addBinding().to(UpdateAlertScriptPaths.class);
catalogBinder.addBinding().to(FinalUpgradeCatalog.class);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog271.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog271.java
new file mode 100644
index 0000000..c8a4d99
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog271.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.upgrade;
+
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.orm.dao.DaoUtils;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+
+public class UpgradeCatalog271 extends AbstractUpgradeCatalog {
+
+ /**
+ * Logger
+ */
+ private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog271.class);
+
+ @Inject
+ DaoUtils daoUtils;
+
+ /**
+ * Constructor
+ *
+ * @param injector
+ */
+ @Inject
+ public UpgradeCatalog271(Injector injector) {
+ super(injector);
+ daoUtils = injector.getInstance(DaoUtils.class);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String getTargetVersion() {
+ return "2.7.1";
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String getSourceVersion() {
+ return "2.7.0";
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void executeDDLUpdates() throws AmbariException, SQLException {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void executePreDMLUpdates() throws AmbariException, SQLException {
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void executeDMLUpdates() throws AmbariException, SQLException {
+ addNewConfigurationsFromXml();
+ updateRangerLogDirConfigs();
+ updateRangerKmsDbUrl();
+ }
+
+ /**
+ * Updating Ranger Admin and Ranger Usersync log directory configs
+ * @throws AmbariException
+ */
+ protected void updateRangerLogDirConfigs() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+ Set<String> installedServices = cluster.getServices().keySet();
+ if (installedServices.contains("RANGER")) {
+ Config rangerEnvConfig = cluster.getDesiredConfigByType("ranger-env");
+ Config rangerAdminSiteConfig = cluster.getDesiredConfigByType("ranger-admin-site");
+ Config rangerUgsyncSiteConfig = cluster.getDesiredConfigByType("ranger-ugsync-site");
+ if (rangerEnvConfig != null) {
+ String rangerAdminLogDir = rangerEnvConfig.getProperties().get("ranger_admin_log_dir");
+ String rangerUsersyncLogDir = rangerEnvConfig.getProperties().get("ranger_usersync_log_dir");
+ if (rangerAdminLogDir != null && rangerAdminSiteConfig != null) {
+ Map<String, String> newProperty = new HashMap<String, String>();
+ newProperty.put("ranger.logs.base.dir", rangerAdminLogDir);
+ updateConfigurationPropertiesForCluster(cluster, "ranger-admin-site", newProperty, true, false);
+ }
+ if (rangerUsersyncLogDir != null && rangerUgsyncSiteConfig != null && rangerUgsyncSiteConfig.getProperties().containsKey("ranger.usersync.logdir")) {
+ Map<String, String> updateProperty = new HashMap<String, String>();
+ updateProperty.put("ranger.usersync.logdir", rangerUsersyncLogDir);
+ updateConfigurationPropertiesForCluster(cluster, "ranger-ugsync-site", updateProperty, true, false);
+ }
+ Set<String> removeProperties = Sets.newHashSet("ranger_admin_log_dir", "ranger_usersync_log_dir");
+ removeConfigurationPropertiesFromCluster(cluster, "ranger-env", removeProperties);
+ }
+ }
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Updating JDBC connection url in Ranger KMS for verifying communication to database
+ * using database root user credentials
+ * @throws AmbariException
+ */
+ protected void updateRangerKmsDbUrl() throws AmbariException {
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ Clusters clusters = ambariManagementController.getClusters();
+ if (clusters != null) {
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+ if (clusterMap != null && !clusterMap.isEmpty()) {
+ for (final Cluster cluster : clusterMap.values()) {
+ Set<String> installedServices = cluster.getServices().keySet();
+ if (installedServices.contains("RANGER_KMS")) {
+ Config rangerKmsPropertiesConfig = cluster.getDesiredConfigByType("kms-properties");
+ Config rangerKmsEnvConfig = cluster.getDesiredConfigByType("kms-env");
+ Config rangerKmsDbksConfig = cluster.getDesiredConfigByType("dbks-site");
+ if (rangerKmsPropertiesConfig != null) {
+ String dbFlavor = rangerKmsPropertiesConfig.getProperties().get("DB_FLAVOR");
+ String dbHost = rangerKmsPropertiesConfig.getProperties().get("db_host");
+ String rangerKmsRootDbUrl = "";
+ if (dbFlavor != null && dbHost != null) {
+ String port = "";
+ if (rangerKmsDbksConfig != null) {
+ String rangerKmsDbUrl = rangerKmsDbksConfig.getProperties().get("ranger.ks.jpa.jdbc.url");
+ if (rangerKmsDbUrl != null) {
+ Pattern pattern = Pattern.compile("(:[0-9]+)");
+ Matcher matcher = pattern.matcher(rangerKmsDbUrl);
+ if (matcher.find()) {
+ port = matcher.group();
+ }
+ }
+ }
+ if ("MYSQL".equalsIgnoreCase(dbFlavor)) {
+ rangerKmsRootDbUrl = "jdbc:mysql://" + dbHost + (!port.equalsIgnoreCase("")?port:":3306");
+ } else if ("ORACLE".equalsIgnoreCase(dbFlavor)) {
+ rangerKmsRootDbUrl = "jdbc:oracle:thin:@//" + dbHost + (!port.equalsIgnoreCase("")?port:":1521");
+ } else if ("POSTGRES".equalsIgnoreCase(dbFlavor)) {
+ rangerKmsRootDbUrl = "jdbc:postgresql://" + dbHost + (!port.equalsIgnoreCase("")?port:":5432") + "/postgres";
+ } else if ("MSSQL".equalsIgnoreCase(dbFlavor)) {
+ rangerKmsRootDbUrl = "jdbc:sqlserver://" + dbHost + (!port.equalsIgnoreCase("")?port:":1433");
+ } else if ("SQLA".equalsIgnoreCase(dbFlavor)) {
+ rangerKmsRootDbUrl = "jdbc:sqlanywhere:host=" + dbHost + (!port.equalsIgnoreCase("")?port:":2638") + ";";
+ }
+ Map<String, String> newProperty = new HashMap<String, String>();
+ newProperty.put("ranger_kms_privelege_user_jdbc_url", rangerKmsRootDbUrl);
+ if (rangerKmsEnvConfig != null) {
+ updateConfigurationPropertiesForCluster(cluster, "kms-env", newProperty, true, false);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
index 7b2de27..5c2b60e 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
@@ -209,8 +209,6 @@ xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFI
# ambari-server hostname
ambari_server_hostname = config['ambariLevelParams']['ambari_server_host']
-ranger_admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir","/var/log/ranger/admin")
-
# ranger kafka plugin enabled property
enable_ranger_kafka = default("configurations/ranger-kafka-plugin-properties/ranger-kafka-plugin-enabled", "No")
enable_ranger_kafka = True if enable_ranger_kafka.lower() == 'yes' else False
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
index a65a53c..206d820 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py
@@ -128,8 +128,10 @@ ambari_java_home = default("/ambariLevelParams/ambari_java_home", java_home)
unix_user = config['configurations']['ranger-env']['ranger_user']
unix_group = config['configurations']['ranger-env']['ranger_group']
ranger_pid_dir = default("/configurations/ranger-env/ranger_pid_dir", "/var/run/ranger")
-usersync_log_dir = default("/configurations/ranger-env/ranger_usersync_log_dir", "/var/log/ranger/usersync")
-admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin")
+old_usersync_log_dir = default("/configurations/ranger-env/ranger_usersync_log_dir", "/var/log/ranger/usersync")
+old_admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin")
+usersync_log_dir = default("/configurations/ranger-ugsync-site/ranger.usersync.logdir", old_usersync_log_dir)
+admin_log_dir = default("/configurations/ranger-admin-site/ranger.logs.base.dir", old_admin_log_dir)
ranger_admin_default_file = format('{ranger_conf}/ranger-admin-default-site.xml')
security_app_context_file = format('{ranger_conf}/security-applicationContext.xml')
ranger_ugsync_default_file = format('{ranger_ugsync_conf}/ranger-ugsync-default.xml')
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2 b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2
index 6c5bb1f..9724cb6 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2
@@ -20,17 +20,17 @@
{
"type":"ranger_admin",
"rowtype":"service",
- "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/xa_portal.log"
+ "path":"{{default('/configurations/ranger-admin-site/ranger.logs.base.dir', '/var/log/ranger/admin')}}/xa_portal.log"
},
{
"type":"ranger_dbpatch",
"is_enabled":"true",
- "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/ranger_db_patch.log"
+ "path":"{{default('/configurations/ranger-admin-site/ranger.logs.base.dir', '/var/log/ranger/admin')}}/ranger_db_patch.log"
},
{
"type":"ranger_usersync",
"rowtype":"service",
- "path":"{{default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync')}}/usersync.log"
+ "path":"{{default('/configurations/ranger-ugsync-site/ranger.usersync.logdir', '/var/log/ranger/usersync')}}/usersync.log"
}
],
"filter":[
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.5.0/configuration/ranger-ugsync-site.xml b/ambari-server/src/main/resources/common-services/RANGER/0.5.0/configuration/ranger-ugsync-site.xml
index 331d07b..cbe98a9 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.5.0/configuration/ranger-ugsync-site.xml
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.5.0/configuration/ranger-ugsync-site.xml
@@ -308,10 +308,10 @@
</property>
<property>
<name>ranger.usersync.logdir</name>
- <value>{{usersync_log_dir}}</value>
+ <display-name>Ranger Usersync Log Dir</display-name>
+ <value>/var/log/ranger/usersync</value>
<description>User sync log directory</description>
<value-attributes>
- <visible>false</visible>
<overridable>false</overridable>
</value-attributes>
<on-ambari-upgrade add="true"/>
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-admin-site.xml b/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-admin-site.xml
index ebf8517..94eb2f0 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-admin-site.xml
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-admin-site.xml
@@ -28,4 +28,15 @@
<description></description>
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>ranger.logs.base.dir</name>
+ <display-name>Ranger Admin Log Dir</display-name>
+ <value>/var/log/ranger/admin</value>
+ <description>Ranger Admin Log Dir</description>
+ <value-attributes>
+ <type>directory</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
</configuration>
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-env.xml b/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-env.xml
index dfc9ca9..86e2204 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-env.xml
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-env.xml
@@ -70,4 +70,14 @@
</value-attributes>
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>ranger_usersync_log_dir</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ranger_admin_log_dir</name>
+ <deleted>true</deleted>
+ <on-ambari-upgrade add="false"/>
+ </property>
</configuration>
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml
index fbb5574..6bb53c1 100644
--- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml
+++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml
@@ -101,4 +101,24 @@
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
+ <property>
+ <name>ranger_kms_privelege_user_jdbc_url</name>
+ <display-name>JDBC connect string for root user</display-name>
+ <description>JDBC connect string - auto populated based on other values. This is to be used by root user</description>
+ <value>jdbc:mysql://localhost</value>
+ <value-attributes>
+ <overridable>false</overridable>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>kms-properties</type>
+ <name>DB_FLAVOR</name>
+ </property>
+ <property>
+ <type>kms-properties</type>
+ <name>db_host</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
</configuration>
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 3fbb1e4..5b4177c 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -453,8 +453,6 @@ if dfs_ha_namenode_active is not None:
else:
namenode_hostname = config['clusterHostInfo']['namenode_hosts'][0]
-ranger_admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir","/var/log/ranger/admin")
-
scheme = 'http' if not yarn_https_on else 'https'
yarn_rm_address = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'] if not yarn_https_on else config['configurations']['yarn-site']['yarn.resourcemanager.webapp.https.address']
rm_active_port = rm_https_port if yarn_https_on else rm_port
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json
index c08a56c..34f9167 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json
@@ -167,14 +167,6 @@
"subsection-name": "subsection-kms-db-row1-col2"
},
{
- "config": "kms-properties/db_root_user",
- "subsection-name": "subsection-kms-db-root-user-row3-col1"
- },
- {
- "config": "kms-properties/db_root_password",
- "subsection-name": "subsection-kms-db-root-user-row3-col2"
- },
- {
"config": "kms-properties/KMS_MASTER_KEY_PASSWD",
"subsection-name": "subsection-kms-master-row4-col"
},
@@ -206,6 +198,25 @@
}
}
]
+ },
+ {
+ "config": "kms-properties/db_root_user",
+ "subsection-name": "subsection-kms-db-root-user-row3-col1"
+ },
+ {
+ "config": "kms-env/ranger_kms_privelege_user_jdbc_url",
+ "subsection-name": "subsection-kms-db-root-user-row3-col1"
+ },
+ {
+ "config": "kms-properties/db_root_password",
+ "subsection-name": "subsection-kms-db-root-user-row3-col2"
+ },
+ {
+ "config": "kms-env/test_root_db_kms_connection",
+ "subsection-name": "subsection-kms-db-root-user-row3-col1",
+ "property_value_attributes": {
+ "ui_only_property": true
+ }
}
]
},
@@ -287,6 +298,28 @@
}
},
{
+ "config": "kms-env/test_root_db_kms_connection",
+ "widget": {
+ "type": "test-db-connection",
+ "display-name": "Test Connection",
+ "required-properties": {
+ "jdbc.driver.class": "dbks-site/ranger.ks.jpa.jdbc.driver",
+ "jdbc.driver.url": "kms-env/ranger_kms_privelege_user_jdbc_url",
+ "db.connection.source.host": "ranger_kms-site/ranger_kms_server_hosts",
+ "db.type": "kms-properties/DB_FLAVOR",
+ "db.connection.destination.host": "kms-properties/db_host",
+ "db.connection.user": "kms-properties/db_root_user",
+ "db.connection.password": "kms-properties/db_root_password"
+ }
+ }
+ },
+ {
+ "config": "kms-env/ranger_kms_privelege_user_jdbc_url",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
"config": "dbks-site/ranger.ks.jpa.jdbc.driver",
"widget" : {
"type": "text-field"
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index c4c493d..ac4632b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -443,6 +443,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
security_enabled = self.isSecurityEnabled(services)
putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services)
putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site")
+ putRangerKmsEnvProperty = self.putProperty(configurations, "kms-env", services)
if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']):
@@ -470,6 +471,24 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
for key in rangerKmsDbProperties:
putRangerKmsDbksProperty(key, rangerKmsDbProperties.get(key))
+ if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']) \
+ and ('db_host' in services['configurations']['kms-properties']['properties']):
+
+ rangerKmsDbFlavor = services['configurations']["kms-properties"]["properties"]["DB_FLAVOR"]
+ rangerKmsDbHost = services['configurations']["kms-properties"]["properties"]["db_host"]
+
+ ranger_kms_db_privelege_url_dict = {
+ 'MYSQL': {'ranger_kms_privelege_user_jdbc_url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost)},
+ 'ORACLE': {'ranger_kms_privelege_user_jdbc_url': 'jdbc:oracle:thin:@' + self.getOracleDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost, None)},
+ 'POSTGRES': {'ranger_kms_privelege_user_jdbc_url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/postgres'},
+ 'MSSQL': {'ranger_kms_privelege_user_jdbc_url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';'},
+ 'SQLA': {'ranger_kms_privelege_user_jdbc_url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';'}
+ }
+
+ rangerKmsPrivelegeDbProperties = ranger_kms_db_privelege_url_dict.get(rangerKmsDbFlavor, ranger_kms_db_privelege_url_dict['MYSQL'])
+ for key in rangerKmsPrivelegeDbProperties:
+ putRangerKmsEnvProperty(key, rangerKmsPrivelegeDbProperties.get(key))
+
if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList:
kmsUser = kmsEnvProperties['kms_user']
kmsUserOld = self.getOldValue(services, 'kms-env', 'kms_user')
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog271Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog271Test.java
new file mode 100644
index 0000000..d7a1069
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog271Test.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.upgrade;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createMockBuilder;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+import java.lang.reflect.Method;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.AmbariManagementControllerImpl;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.StackId;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.easymock.EasyMockSupport;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.inject.Injector;
+
+public class UpgradeCatalog271Test {
+
+ @Test
+ public void testExecuteDMLUpdates() throws Exception {
+ Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml");
+ Method updateRangerLogDirConfigs = UpgradeCatalog271.class.getDeclaredMethod("updateRangerLogDirConfigs");
+ Method updateRangerKmsDbUrl = UpgradeCatalog271.class.getDeclaredMethod("updateRangerKmsDbUrl");
+
+ UpgradeCatalog271 upgradeCatalog271 = createMockBuilder(UpgradeCatalog271.class)
+ .addMockedMethod(updateRangerKmsDbUrl)
+ .addMockedMethod(updateRangerLogDirConfigs)
+ .addMockedMethod(addNewConfigurationsFromXml)
+ .createMock();
+
+ upgradeCatalog271.addNewConfigurationsFromXml();
+ expectLastCall().once();
+
+ upgradeCatalog271.updateRangerLogDirConfigs();
+ expectLastCall().once();
+
+ upgradeCatalog271.updateRangerKmsDbUrl();
+ expectLastCall().once();
+
+ replay(upgradeCatalog271);
+ upgradeCatalog271.executeDMLUpdates();
+ verify(upgradeCatalog271);
+ }
+
+ @Test
+ public void testUpdateRangerLogDirConfigs() throws Exception {
+
+ Map<String, Service> installedServices = new HashMap<String, Service>() {
+ {
+ put("RANGER", null);
+ }
+ };
+
+ Map<String, String> rangerEnvConfig = new HashMap<String, String>() {
+ {
+ put("ranger_admin_log_dir", "/var/log/ranger/admin");
+ put("ranger_usersync_log_dir", "/var/log/ranger/usersync");
+ }
+ };
+
+ Map<String, String> oldRangerUgsyncSiteConfig = new HashMap<String, String>() {
+ {
+ put("ranger.usersync.logdir", "{{usersync_log_dir}}");
+ }
+ };
+
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ Clusters clusters = easyMockSupport.createNiceMock(Clusters.class);
+ final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
+
+ Injector injector = easyMockSupport.createNiceMock(Injector.class);
+ AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
+ .addMockedMethod("createConfiguration")
+ .addMockedMethod("getClusters", new Class[] { })
+ .addMockedMethod("createConfig")
+ .createNiceMock();
+
+ expect(injector.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
+ expect(controller.getClusters()).andReturn(clusters).anyTimes();
+
+ expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", cluster);
+ }}).once();
+ expect(cluster.getClusterName()).andReturn("cl1").anyTimes();
+ expect(cluster.getServices()).andReturn(installedServices).atLeastOnce();
+
+ Config mockRangerEnvConfig = easyMockSupport.createNiceMock(Config.class);
+ expect(cluster.getDesiredConfigByType("ranger-env")).andReturn(mockRangerEnvConfig).atLeastOnce();
+ expect(mockRangerEnvConfig.getProperties()).andReturn(rangerEnvConfig).anyTimes();
+
+ Config mockRangerAdminSiteConfig = easyMockSupport.createNiceMock(Config.class);
+ expect(cluster.getDesiredConfigByType("ranger-admin-site")).andReturn(mockRangerAdminSiteConfig).atLeastOnce();
+ expect(mockRangerAdminSiteConfig.getProperties()).andReturn(Collections.emptyMap()).anyTimes();
+
+ Config mockRangerUgsyncSiteConfig = easyMockSupport.createNiceMock(Config.class);
+ expect(cluster.getDesiredConfigByType("ranger-ugsync-site")).andReturn(mockRangerUgsyncSiteConfig).atLeastOnce();
+ expect(mockRangerUgsyncSiteConfig.getProperties()).andReturn(oldRangerUgsyncSiteConfig).anyTimes();
+
+ Capture<Map> rangerAdminpropertiesCapture = EasyMock.newCapture();
+ expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(rangerAdminpropertiesCapture), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+
+ Capture<Map> rangerUgsyncPropertiesCapture = EasyMock.newCapture();
+ expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(rangerUgsyncPropertiesCapture), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+
+ Capture<Map> rangerEnvPropertiesCapture = EasyMock.newCapture();
+ expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(rangerEnvPropertiesCapture), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+
+ replay(controller, injector, clusters, mockRangerEnvConfig, mockRangerAdminSiteConfig, mockRangerUgsyncSiteConfig, cluster);
+ new UpgradeCatalog271(injector).updateRangerLogDirConfigs();
+ easyMockSupport.verifyAll();
+
+ Map<String, String> updatedRangerAdminConfig = rangerAdminpropertiesCapture.getValue();
+ Assert.assertEquals(updatedRangerAdminConfig.get("ranger.logs.base.dir"), "/var/log/ranger/admin");
+
+ Map<String, String> updatedRangerUgsyncSiteConfig = rangerUgsyncPropertiesCapture.getValue();
+ Assert.assertEquals(updatedRangerUgsyncSiteConfig.get("ranger.usersync.logdir"), "/var/log/ranger/usersync");
+
+ Map<String, String> updatedRangerEnvConfig = rangerEnvPropertiesCapture.getValue();
+ Assert.assertFalse(updatedRangerEnvConfig.containsKey("ranger_admin_log_dir"));
+ Assert.assertFalse(updatedRangerEnvConfig.containsKey("ranger_usersync_log_dir"));
+ }
+
+ @Test
+ public void testUpdateRangerKmsDbUrl() throws Exception {
+
+ Map<String, Service> installedServices = new HashMap<String, Service>() {
+ {
+ put("RANGER_KMS", null);
+ }
+ };
+
+ Map<String, String> rangerKmsPropertiesConfig = new HashMap<String, String>() {
+ {
+ put("DB_FLAVOR", "MYSQL");
+ put("db_host", "c6401.ambari.apache.org");
+ }
+ };
+
+ Map<String, String> rangerKmsDbksPropertiesConfig = new HashMap<String, String>() {
+ {
+ put("ranger.ks.jpa.jdbc.url", "jdbc:mysql://c6401.ambari.apache.org:3546");
+ }
+ };
+
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ Clusters clusters = easyMockSupport.createNiceMock(Clusters.class);
+ final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
+
+ Injector injector = easyMockSupport.createNiceMock(Injector.class);
+ AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
+ .addMockedMethod("createConfiguration")
+ .addMockedMethod("getClusters", new Class[] { })
+ .addMockedMethod("createConfig")
+ .createNiceMock();
+
+ expect(injector.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
+ expect(controller.getClusters()).andReturn(clusters).anyTimes();
+
+ expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+ put("normal", cluster);
+ }}).once();
+ expect(cluster.getClusterName()).andReturn("cl1").once();
+ expect(cluster.getServices()).andReturn(installedServices).atLeastOnce();
+
+ Config mockRangerKmsPropertiesConfig = easyMockSupport.createNiceMock(Config.class);
+ expect(cluster.getDesiredConfigByType("kms-properties")).andReturn(mockRangerKmsPropertiesConfig).atLeastOnce();
+
+ Config mockRangerKmsEnvConfig = easyMockSupport.createNiceMock(Config.class);
+ expect(cluster.getDesiredConfigByType("kms-env")).andReturn(mockRangerKmsEnvConfig).atLeastOnce();
+
+ Config mockRangerKmsDbksConfig = easyMockSupport.createNiceMock(Config.class);
+ expect(cluster.getDesiredConfigByType("dbks-site")).andReturn(mockRangerKmsDbksConfig).atLeastOnce();
+
+ expect(mockRangerKmsPropertiesConfig.getProperties()).andReturn(rangerKmsPropertiesConfig).anyTimes();
+ expect(mockRangerKmsEnvConfig.getProperties()).andReturn(Collections.emptyMap()).anyTimes();
+ expect(mockRangerKmsDbksConfig.getProperties()).andReturn(rangerKmsDbksPropertiesConfig).anyTimes();
+
+ Capture<Map> propertiesCapture = EasyMock.newCapture();
+ expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(propertiesCapture), anyString(),
+ anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+
+ replay(controller, injector, clusters, mockRangerKmsPropertiesConfig, mockRangerKmsEnvConfig, mockRangerKmsDbksConfig, cluster);
+ new UpgradeCatalog271(injector).updateRangerKmsDbUrl();
+ easyMockSupport.verifyAll();
+
+ Map<String, String> updatedRangerKmsEnvConfig = propertiesCapture.getValue();
+ Assert.assertEquals(updatedRangerKmsEnvConfig.get("ranger_kms_privelege_user_jdbc_url"), "jdbc:mysql://c6401.ambari.apache.org:3546");
+ }
+
+}
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 6539302..6dd94d9 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1698,6 +1698,11 @@ class TestHDP23StackAdvisor(TestCase):
'hadoop.kms.proxyuser.root.hosts': {'delete': 'true'},
'hadoop.kms.proxyuser.root.users': {'delete': 'true'}
}
+ },
+ 'kms-env': {
+ 'properties': {
+ 'ranger_kms_privelege_user_jdbc_url': 'jdbc:oracle:thin:@c6401.ambari.apache.org:1521:XE'
+ }
}
}
@@ -1741,6 +1746,11 @@ class TestHDP23StackAdvisor(TestCase):
'hadoop.kms.proxyuser.ambari-cl1.hosts': '*',
'hadoop.kms.proxyuser.ambari-cl1.users': '*'
}
+ },
+ 'kms-env': {
+ 'properties': {
+ 'ranger_kms_privelege_user_jdbc_url': 'jdbc:oracle:thin:@c6401.ambari.apache.org:1521:XE'
+ }
}
}
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json
index 91a9b79..53e66ed 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json
@@ -534,7 +534,7 @@
"ranger.usersync.ldap.referral": "ignore",
"ranger.usersync.group.searchfilter": "",
"ranger.usersync.ldap.user.objectclass": "person",
- "ranger.usersync.logdir": "{{usersync_log_dir}}",
+ "ranger.usersync.logdir": "/var/log/ranger/usersync",
"ranger.usersync.ldap.user.searchfilter": "",
"ranger.usersync.ldap.groupname.caseconversion": "none",
"ranger.usersync.ldap.ldapbindpassword": "",
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json
index 05edf27..3719a8d 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json
@@ -586,7 +586,7 @@
"ranger.usersync.ldap.referral": "ignore",
"ranger.usersync.group.searchfilter": "",
"ranger.usersync.ldap.user.objectclass": "person",
- "ranger.usersync.logdir": "{{usersync_log_dir}}",
+ "ranger.usersync.logdir": "/var/log/ranger/usersync",
"ranger.usersync.ldap.user.searchfilter": "",
"ranger.usersync.ldap.groupname.caseconversion": "none",
"ranger.usersync.ldap.ldapbindpassword": "",
diff --git a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json
index 43d65a0..7888054 100644
--- a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json
+++ b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json
@@ -565,7 +565,7 @@
"ranger.usersync.ldap.referral": "ignore",
"ranger.usersync.group.searchfilter": "",
"ranger.usersync.ldap.user.objectclass": "person",
- "ranger.usersync.logdir": "{{usersync_log_dir}}",
+ "ranger.usersync.logdir": "/var/log/ranger/usersync",
"ranger.usersync.ldap.user.searchfilter": "",
"ranger.usersync.ldap.groupname.caseconversion": "none",
"ranger.usersync.ldap.ldapbindpassword": "",
diff --git a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json
index 3fc0316..8228401 100644
--- a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json
+++ b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json
@@ -615,7 +615,7 @@
"ranger.usersync.ldap.referral": "ignore",
"ranger.usersync.group.searchfilter": "",
"ranger.usersync.ldap.user.objectclass": "person",
- "ranger.usersync.logdir": "{{usersync_log_dir}}",
+ "ranger.usersync.logdir": "/var/log/ranger/usersync",
"ranger.usersync.ldap.user.searchfilter": "",
"ranger.usersync.ldap.groupname.caseconversion": "none",
"ranger.usersync.ldap.ldapbindpassword": "",